From f65223137e8bdf7d18f3e60d64022d2ff9196b1b Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 7 Apr 2026 00:49:08 +0200 Subject: [PATCH] automation unification implemented --- app.py | 22 +- env_dev.env | 2 +- .../automation/datamodelFeatureAutomation.py | 97 - .../automation/interfaceFeatureAutomation.py | 872 ----- modules/features/automation/mainAutomation.py | 446 --- .../automation/routeFeatureAutomation.py | 1264 ------- .../automation/subAutomationTemplates.py | 433 --- .../features/automation/subAutomationUtils.py | 118 - modules/features/automation2/__init__.py | 2 - .../datamodelFeatureAutomation2.py | 166 - modules/features/graphicalEditor/__init__.py | 2 + .../datamodelFeatureGraphicalEditor.py | 482 +++ .../emailPoller.py | 4 +- .../entryPoints.py | 0 .../interfaceFeatureGraphicalEditor.py} | 242 +- .../mainGraphicalEditor.py} | 104 +- .../nodeDefinitions/__init__.py | 0 .../nodeDefinitions/ai.py | 0 .../nodeDefinitions/clickup.py | 0 .../nodeDefinitions/email.py | 0 .../nodeDefinitions/file.py | 0 .../nodeDefinitions/flow.py | 0 .../nodeDefinitions/input.py | 0 .../nodeDefinitions/sharepoint.py | 0 .../nodeDefinitions/triggers.py | 0 .../nodeRegistry.py | 4 +- .../routeFeatureGraphicalEditor.py} | 645 +++- modules/interfaces/interfaceBootstrap.py | 255 +- modules/interfaces/interfaceRbac.py | 14 +- modules/routes/routeAdminAutomationEvents.py | 285 -- modules/routes/routeAdminAutomationLogs.py | 207 -- modules/routes/routeAdminFeatures.py | 72 +- modules/routes/routeInvitations.py | 33 +- modules/routes/routeSystem.py | 7 +- .../serviceAgent/coreTools/__init__.py | 7 + .../coreTools/_connectionTools.py | 193 ++ .../coreTools/_crossWorkflowTools.py | 175 + .../coreTools/_dataSourceTools.py | 258 ++ .../serviceAgent/coreTools/_documentTools.py | 374 ++ .../coreTools/_featureSubAgentTools.py | 160 + .../serviceAgent/coreTools/_helpers.py | 62 + .../serviceAgent/coreTools/_mediaTools.py | 958 ++++++ .../serviceAgent/coreTools/_workspaceTools.py | 950 ++++++ .../serviceAgent/coreTools/registerCore.py | 27 + .../services/serviceAgent/datamodelAgent.py | 2 + .../services/serviceAgent/mainServiceAgent.py | 3015 +---------------- .../services/serviceAgent/toolboxRegistry.py | 183 + .../services/serviceAgent/workflowTools.py | 479 +++ ...subSubscriptionGraphicalEditorRunFailed.py | 71 + modules/system/mainSystem.py | 37 - modules/workflows/automation/__init__.py | 10 - modules/workflows/automation/mainWorkflow.py | 325 -- .../automation/subAutomationSchedule.py | 65 - .../automation/subAutomationTemplates.py | 385 --- .../automation/subAutomationUtils.py | 110 - .../workflows/automation2/executionEngine.py | 165 +- .../executors/actionNodeExecutor.py | 4 +- .../automation2/subAutomation2Schedule.py | 8 +- modules/workflows/scheduler/__init__.py | 2 + modules/workflows/scheduler/mainScheduler.py | 459 +++ scripts/migrate_async_to_sync.py | 6 - scripts/script_db_export_migration.py | 4 - .../test05_workflow_with_documents.py | 369 -- .../test06_workflow_prompt_variations.py | 471 --- .../test09_document_generation_formats.py | 735 ---- .../test10_document_generation_formats.py | 558 --- .../test11_code_generation_formats.py | 559 --- 67 files changed, 5979 insertions(+), 10985 deletions(-) delete mode 100644 modules/features/automation/datamodelFeatureAutomation.py delete mode 100644 modules/features/automation/interfaceFeatureAutomation.py delete mode 100644 modules/features/automation/mainAutomation.py delete mode 100644 modules/features/automation/routeFeatureAutomation.py delete mode 100644 modules/features/automation/subAutomationTemplates.py delete mode 100644 modules/features/automation/subAutomationUtils.py delete mode 100644 modules/features/automation2/__init__.py delete mode 100644 modules/features/automation2/datamodelFeatureAutomation2.py create mode 100644 modules/features/graphicalEditor/__init__.py create mode 100644 modules/features/graphicalEditor/datamodelFeatureGraphicalEditor.py rename modules/features/{automation2 => graphicalEditor}/emailPoller.py (97%) rename modules/features/{automation2 => graphicalEditor}/entryPoints.py (100%) rename modules/features/{automation2/interfaceFeatureAutomation2.py => graphicalEditor/interfaceFeatureGraphicalEditor.py} (62%) rename modules/features/{automation2/mainAutomation2.py => graphicalEditor/mainGraphicalEditor.py} (70%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/__init__.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/ai.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/clickup.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/email.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/file.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/flow.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/input.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/sharepoint.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeDefinitions/triggers.py (100%) rename modules/features/{automation2 => graphicalEditor}/nodeRegistry.py (94%) rename modules/features/{automation2/routeFeatureAutomation2.py => graphicalEditor/routeFeatureGraphicalEditor.py} (54%) delete mode 100644 modules/routes/routeAdminAutomationEvents.py delete mode 100644 modules/routes/routeAdminAutomationLogs.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/__init__.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_connectionTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_crossWorkflowTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_documentTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_helpers.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/_workspaceTools.py create mode 100644 modules/serviceCenter/services/serviceAgent/coreTools/registerCore.py create mode 100644 modules/serviceCenter/services/serviceAgent/toolboxRegistry.py create mode 100644 modules/serviceCenter/services/serviceAgent/workflowTools.py create mode 100644 modules/serviceCenter/services/serviceMessaging/subscriptions/subSubscriptionGraphicalEditorRunFailed.py delete mode 100644 modules/workflows/automation/__init__.py delete mode 100644 modules/workflows/automation/mainWorkflow.py delete mode 100644 modules/workflows/automation/subAutomationSchedule.py delete mode 100644 modules/workflows/automation/subAutomationTemplates.py delete mode 100644 modules/workflows/automation/subAutomationUtils.py create mode 100644 modules/workflows/scheduler/__init__.py create mode 100644 modules/workflows/scheduler/mainScheduler.py delete mode 100644 tests/functional/test05_workflow_with_documents.py delete mode 100644 tests/functional/test06_workflow_prompt_variations.py delete mode 100644 tests/functional/test09_document_generation_formats.py delete mode 100644 tests/functional/test10_document_generation_formats.py delete mode 100644 tests/functional/test11_code_generation_formats.py diff --git a/app.py b/app.py index f29436cc..9627a698 100644 --- a/app.py +++ b/app.py @@ -20,10 +20,8 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager -from modules.workflows.automation import subAutomationSchedule -from modules.workflows.automation2 import subAutomation2Schedule -from modules.features.automation2.emailPoller import start as startAutomation2EmailPoller -from modules.features.automation2.emailPoller import stop as stopAutomation2EmailPoller +from modules.features.graphicalEditor.emailPoller import start as startGraphicalEditorEmailPoller +from modules.features.graphicalEditor.emailPoller import stop as stopGraphicalEditorEmailPoller from modules.interfaces.interfaceDbApp import getRootInterface from modules.system.registry import loadFeatureMainModules @@ -357,15 +355,14 @@ async def lifespan(app: FastAPI): # --- Init Managers --- import asyncio + from modules.workflows.automation2 import subAutomation2Schedule try: main_loop = asyncio.get_running_loop() eventManager.set_event_loop(main_loop) subAutomation2Schedule.set_main_loop(main_loop) except RuntimeError: pass - subAutomationSchedule.start(eventUser) # Automation scheduler - subAutomation2Schedule.start(eventUser) # Automation2 schedule trigger (cron) - # Automation2 email poller: started on-demand when a run pauses for email.checkEmail + subAutomation2Schedule.start(eventUser) eventManager.start() # Register audit log cleanup scheduler @@ -394,10 +391,9 @@ async def lifespan(app: FastAPI): yield # --- Stop Managers --- - stopAutomation2EmailPoller(eventUser) # Automation2 email poller (no-op if not running) - subAutomation2Schedule.stop(eventUser) # Automation2 schedule + stopGraphicalEditorEmailPoller(eventUser) + subAutomation2Schedule.stop(eventUser) eventManager.stop() - subAutomationSchedule.stop(eventUser) # Automation scheduler # --- Stop Feature Containers (Plug&Play) --- try: @@ -592,12 +588,6 @@ app.include_router(adminSecurityRouter) from modules.routes.routeSharepoint import router as sharepointRouter app.include_router(sharepointRouter) -from modules.routes.routeAdminAutomationEvents import router as adminAutomationEventsRouter -app.include_router(adminAutomationEventsRouter) - -from modules.routes.routeAdminAutomationLogs import router as adminAutomationLogsRouter -app.include_router(adminAutomationLogsRouter) - from modules.routes.routeAdminLogs import router as adminLogsRouter app.include_router(adminLogsRouter) diff --git a/env_dev.env b/env_dev.env index 30ffd079..9c13506f 100644 --- a/env_dev.env +++ b/env_dev.env @@ -4,7 +4,7 @@ APP_ENV_TYPE = dev APP_ENV_LABEL = Development Instance Patrick APP_API_URL = http://localhost:8000 -APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/key.txt +APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/notes/key.txt APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9 APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9 diff --git a/modules/features/automation/datamodelFeatureAutomation.py b/modules/features/automation/datamodelFeatureAutomation.py deleted file mode 100644 index 8ea4a300..00000000 --- a/modules/features/automation/datamodelFeatureAutomation.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Automation models: AutomationDefinition, AutomationTemplate.""" - -from typing import List, Dict, Any, Optional -from pydantic import BaseModel, Field -from modules.datamodels.datamodelBase import PowerOnModel -from modules.shared.attributeUtils import registerModelLabels -from modules.datamodels.datamodelUtils import TextMultilingual -import uuid - - -class AutomationDefinition(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) - schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ - {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, - {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, - {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} - ]}) - template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) - placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "textarea"}) - active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) - eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - allowedProviders: List[str] = Field(default_factory=list, description="List of allowed AICore providers (e.g., 'anthropic', 'openai'). Empty means all RBAC-permitted providers are allowed.", json_schema_extra={"frontend_type": "multiselect", "frontend_readonly": False, "frontend_required": False}) - - -registerModelLabels( - "AutomationDefinition", - {"en": "Automation Definition", "ge": "Automatisierungs-Definition", "fr": "Définition d'automatisation"}, - { - "id": {"en": "ID", "ge": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "ge": "Mandanten-ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "ge": "Feature-Instanz-ID", "fr": "ID de l'instance de fonctionnalité"}, - "label": {"en": "Label", "ge": "Bezeichnung", "fr": "Libellé"}, - "schedule": {"en": "Schedule", "ge": "Zeitplan", "fr": "Planification"}, - "template": {"en": "Template", "ge": "Vorlage", "fr": "Modèle"}, - "placeholders": {"en": "Placeholders", "ge": "Platzhalter", "fr": "Espaces réservés"}, - "active": {"en": "Active", "ge": "Aktiv", "fr": "Actif"}, - "eventId": {"en": "Event ID", "ge": "Event-ID", "fr": "ID de l'événement"}, - "status": {"en": "Status", "ge": "Status", "fr": "Statut"}, - "executionLogs": {"en": "Execution Logs", "ge": "Ausführungsprotokolle", "fr": "Journaux d'exécution"}, - "allowedProviders": {"en": "Allowed Providers", "ge": "Erlaubte Provider", "fr": "Fournisseurs autorisés"}, - }, -) - - -class AutomationTemplate(PowerOnModel): - """Automation-Vorlage ohne scharfe Placeholder-Werte (DB-persistiert). - - System-Templates (isSystem=True): Nur durch SysAdmin aenderbar. Alle User koennen lesen. - Instance-Templates (isSystem=False, featureInstanceId gesetzt): CRUD durch Instance-Admin/Editor. - """ - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True} - ) - label: TextMultilingual = Field( - description="Template name (multilingual)", - json_schema_extra={"frontend_type": "multilingual", "frontend_required": True} - ) - overview: Optional[TextMultilingual] = Field( - None, - description="Short description (multilingual)", - json_schema_extra={"frontend_type": "multilingual", "frontend_required": False} - ) - template: str = Field( - description="JSON workflow structure with {{KEY:...}} placeholders", - json_schema_extra={"frontend_type": "textarea", "frontend_required": True} - ) - isSystem: bool = Field( - default=False, - description="System template (only SysAdmin can modify, all users can read)", - json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": False} - ) - featureInstanceId: Optional[str] = Field( - None, - description="Feature instance ID (null for system templates, set for instance-scoped templates)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} - ) -registerModelLabels( - "AutomationTemplate", - {"en": "Automation Template", "ge": "Automation-Vorlage", "fr": "Modèle d'automatisation"}, - { - "id": {"en": "ID", "ge": "ID", "fr": "ID"}, - "label": {"en": "Label", "ge": "Bezeichnung", "fr": "Libellé"}, - "overview": {"en": "Overview", "ge": "Übersicht", "fr": "Aperçu"}, - "template": {"en": "Template", "ge": "Vorlage", "fr": "Modèle"}, - "isSystem": {"en": "System Template", "ge": "System-Vorlage", "fr": "Modèle système"}, - "featureInstanceId": {"en": "Feature Instance", "ge": "Feature-Instanz", "fr": "Instance de fonctionnalité"}, - }, -) diff --git a/modules/features/automation/interfaceFeatureAutomation.py b/modules/features/automation/interfaceFeatureAutomation.py deleted file mode 100644 index a4f90a51..00000000 --- a/modules/features/automation/interfaceFeatureAutomation.py +++ /dev/null @@ -1,872 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Interface for Automation feature - manages AutomationDefinition and AutomationTemplate. -Uses the PostgreSQL connector for data access with user/mandate filtering. -""" - -import logging -import uuid -import math -from typing import Dict, Any, List, Optional, Union - -from modules.security.rbac import RbacClass -from modules.datamodels.datamodelRbac import AccessRuleContext -from modules.datamodels.datamodelUam import AccessLevel, User -from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition, AutomationTemplate -from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, buildDataObjectKey - -from modules.shared.configuration import APP_CONFIG - -logger = logging.getLogger(__name__) - - -def _automationDefinitionPayload(data: Dict[str, Any]) -> Dict[str, Any]: - """Strip connector/enrichment keys; only fields defined on AutomationDefinition.""" - allowed = AutomationDefinition.model_fields.keys() - return {k: v for k, v in (data or {}).items() if k in allowed} - - -# Singleton factory for Automation instances -_automationInterfaces = {} - - -class AutomationObjects: - """ - Interface for Automation database operations. - Manages AutomationDefinition and AutomationTemplate with RBAC support. - """ - - def __init__(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - self.currentUser = currentUser - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - self.userId = currentUser.id if currentUser else None - - # Initialize database with proper configuration - self._initializeDatabase() - - # Initialize RBAC - AccessRules are in poweron_app, not poweron_automation! - from modules.security.rootAccess import getRootDbAppConnector - dbApp = getRootDbAppConnector() - self.rbac = RbacClass(self.db, dbApp=dbApp) - - # Update database context - self.db.updateContext(self.userId) - - def _initializeDatabase(self): - """Initializes the database connection with proper configuration.""" - # Get configuration values - dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") - dbDatabase = "poweron_automation" - dbUser = APP_CONFIG.get("DB_USER") - dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) - - # Create database connector with full configuration - self.db = DatabaseConnector( - dbHost=dbHost, - dbDatabase=dbDatabase, - dbUser=dbUser, - dbPassword=dbPassword, - dbPort=dbPort, - userId=self.userId, - ) - - logger.debug(f"Automation database initialized for user {self.userId}") - - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - """Update user context for the interface.""" - self.currentUser = currentUser - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - self.userId = currentUser.id if currentUser else None - if hasattr(self.db, 'updateContext'): - self.db.updateContext(self.userId) - - def checkRbacPermission(self, model, action: str, recordId: str = None) -> bool: - """Check RBAC permission for a specific action on a model.""" - objectKey = buildDataObjectKey(model.__name__) - permissions = self.rbac.getUserPermissions( - user=self.currentUser, - context=AccessRuleContext.DATA, - item=objectKey, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId - ) - - accessLevel = getattr(permissions, action, AccessLevel.NONE) - - if accessLevel == AccessLevel.ALL: - return True - elif accessLevel == AccessLevel.GROUP: - return True - elif accessLevel == AccessLevel.MY: - if recordId: - record = self.db.getRecordset(model, recordFilter={"id": recordId}) - if record: - return record[0].get("sysCreatedBy") == self.userId - else: - return False # Record not found = no access - return True # No recordId needed (e.g., for CREATE) - return False - - # ========================================================================= - # AutomationDefinition CRUD methods - # ========================================================================= - - def _computeAutomationStatus(self, automation: Dict[str, Any]) -> str: - """Compute status field based on eventId presence""" - eventId = automation.get("eventId") - return "Running" if eventId else "Idle" - - def _enrichAutomationsWithUserAndMandate(self, automations: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Batch enrich automations with user names, mandate names and feature instance labels. - Uses direct DB lookup (no RBAC) because this is purely cosmetic enrichment — - the user already has RBAC-verified access to the automations themselves. - """ - if not automations: - return automations - - # Collect all unique IDs - userIds = set() - mandateIds = set() - featureInstanceIds = set() - - for automation in automations: - createdBy = automation.get("sysCreatedBy") - if createdBy: - userIds.add(createdBy) - - mandateId = automation.get("mandateId") - if mandateId: - mandateIds.add(mandateId) - - featureInstanceId = automation.get("featureInstanceId") - if featureInstanceId: - featureInstanceIds.add(featureInstanceId) - - # Use root DB connector for display-only lookups (no RBAC needed) - usersMap = {} - mandatesMap = {} - featureInstancesMap = {} - try: - from modules.datamodels.datamodelUam import UserInDB, Mandate - from modules.datamodels.datamodelFeatures import FeatureInstance - from modules.security.rootAccess import getRootDbAppConnector - dbAppConn = getRootDbAppConnector() - - # Batch fetch user display names - if userIds: - for userId in userIds: - users = dbAppConn.getRecordset(UserInDB, recordFilter={"id": userId}) - if users: - user = users[0] - displayName = user.get("fullName") or user.get("username") or user.get("email") or None - if displayName: - usersMap[userId] = displayName - - # Batch fetch mandate display names - if mandateIds: - for mandateId in mandateIds: - mandates = dbAppConn.getRecordset(Mandate, recordFilter={"id": mandateId}) - if mandates: - label = mandates[0].get("label") or mandates[0].get("name") or None - if label: - mandatesMap[mandateId] = label - - # Batch fetch feature instance labels - if featureInstanceIds: - for fiId in featureInstanceIds: - instances = dbAppConn.getRecordset(FeatureInstance, recordFilter={"id": fiId}) - if instances: - fi = instances[0] - label = fi.get("label") or fi.get("featureCode") or None - if label: - featureInstancesMap[fiId] = label - except Exception as e: - logger.warning(f"Could not enrich automations with display names: {e}") - - # Enrich each automation with the fetched data - # SECURITY: Never show a fallback name — if lookup fails, show empty string - for automation in automations: - createdBy = automation.get("sysCreatedBy") - automation["sysCreatedByUserName"] = usersMap.get(createdBy, "") if createdBy else "" - - mandateId = automation.get("mandateId") - automation["mandateName"] = mandatesMap.get(mandateId, "") if mandateId else "" - - featureInstanceId = automation.get("featureInstanceId") - automation["featureInstanceName"] = featureInstancesMap.get(featureInstanceId, "") if featureInstanceId else "" - - return automations - - def _enrichAutomationWithUserAndMandate(self, automation: Dict[str, Any]) -> Dict[str, Any]: - """ - Enrich a single automation with user name and mandate name for display. - For multiple automations, use _enrichAutomationsWithUserAndMandate for better performance. - """ - return self._enrichAutomationsWithUserAndMandate([automation])[0] - - def getAllAutomationDefinitions(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: - """ - Returns automation definitions based on user access level. - Supports optional pagination, sorting, and filtering. - Computes status field for each automation. - """ - # AutomationDefinitions can belong to any feature instance within a mandate. - # Filter by mandateId only — not by featureInstanceId — to show all definitions across features. - filteredAutomations = getRecordsetWithRBAC( - self.db, - AutomationDefinition, - self.currentUser, - mandateId=self.mandateId - ) - - # Compute status for each automation and normalize executionLogs - for automation in filteredAutomations: - automation["status"] = self._computeAutomationStatus(automation) - # Ensure executionLogs is always a list, not None - if automation.get("executionLogs") is None: - automation["executionLogs"] = [] - - # Batch enrich with user and mandate names - self._enrichAutomationsWithUserAndMandate(filteredAutomations) - - # If no pagination requested, return all items - if pagination is None: - return filteredAutomations - - # Apply filtering (if filters provided) - if pagination.filters: - filteredAutomations = self._applyFilters(filteredAutomations, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination.sort: - filteredAutomations = self._applySorting(filteredAutomations, pagination.sort) - - # Count total items after filters - totalItems = len(filteredAutomations) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedAutomations = filteredAutomations[startIdx:endIdx] - - return PaginatedResult( - items=pagedAutomations, - totalItems=totalItems, - totalPages=totalPages - ) - - def _applyFilters(self, items: List[Dict], filters: Dict[str, Any]) -> List[Dict]: - """Apply filters to a list of items.""" - if not filters: - return items - - filtered = [] - for item in items: - match = True - for key, value in filters.items(): - itemValue = item.get(key) - if isinstance(value, str) and isinstance(itemValue, str): - if value.lower() not in itemValue.lower(): - match = False - break - elif str(itemValue).lower() != str(value).lower(): - match = False - break - if match: - filtered.append(item) - return filtered - - def _applySorting(self, items: List[Dict], sortFields: List[Dict]) -> List[Dict]: - """Apply sorting to a list of items.""" - if not sortFields: - return items - - for sortField in reversed(sortFields): - field = sortField.get("field", "") - direction = sortField.get("direction", "asc") - reverse = direction.lower() == "desc" - items = sorted(items, key=lambda x: x.get(field, ""), reverse=reverse) - - return items - - def getAutomationDefinition(self, automationId: str, includeSystemFields: bool = False) -> Optional[AutomationDefinition]: - """Returns an automation definition by ID if user has access, with computed status. - - Args: - automationId: ID of the automation to get - includeSystemFields: If True, returns raw dict with system fields (sysCreatedBy, etc). - If False (default), returns Pydantic model without system fields. - """ - try: - # AutomationDefinitions can belong to any feature instance within a mandate. - # Filter by mandateId only — not by featureInstanceId. - filtered = getRecordsetWithRBAC( - self.db, - AutomationDefinition, - self.currentUser, - recordFilter={"id": automationId}, - mandateId=self.mandateId - ) - - if not filtered: - return None - - automation = filtered[0] - automation["status"] = self._computeAutomationStatus(automation) - # Ensure executionLogs is always a list, not None - if automation.get("executionLogs") is None: - automation["executionLogs"] = [] - # Enrich with user and mandate names - self._enrichAutomationWithUserAndMandate(automation) - - # For internal use (execution), return raw dict with system fields - if includeSystemFields: - # Return as simple namespace object so getattr works - class AutomationWithSystemFields: - def __init__(self, data): - for key, value in data.items(): - setattr(self, key, value) - return AutomationWithSystemFields(automation) - - # Clean metadata fields and return Pydantic model - cleanedRecord = _automationDefinitionPayload(automation) - return AutomationDefinition(**cleanedRecord) - except Exception as e: - logger.error(f"Error getting automation definition: {str(e)}") - return None - - def createAutomationDefinition(self, automationData: Dict[str, Any]) -> AutomationDefinition: - """Creates a new automation definition, then triggers sync.""" - try: - # Ensure ID is present - if "id" not in automationData or not automationData["id"]: - automationData["id"] = str(uuid.uuid4()) - - # Ensure mandateId and featureInstanceId are set for proper data isolation - if "mandateId" not in automationData or not automationData.get("mandateId"): - # Use request context mandateId, or fall back to Root mandate - effectiveMandateId = self.mandateId - if not effectiveMandateId: - # Fall back to Root mandate (first mandate in system) - try: - from modules.datamodels.datamodelUam import Mandate - from modules.security.rootAccess import getRootDbAppConnector - dbAppConn = getRootDbAppConnector() - allMandates = dbAppConn.getRecordset(Mandate) - if allMandates: - effectiveMandateId = allMandates[0].get("id") - logger.debug(f"createAutomationDefinition: Using Root mandate {effectiveMandateId}") - except Exception as e: - logger.warning(f"Could not get Root mandate: {e}") - automationData["mandateId"] = effectiveMandateId - if "featureInstanceId" not in automationData: - automationData["featureInstanceId"] = self.featureInstanceId - - # Ensure database connector has correct userId context - if not self.userId: - logger.error(f"createAutomationDefinition: userId is not set! Cannot set sysCreatedBy. currentUser={self.currentUser}") - elif hasattr(self.db, 'updateContext'): - try: - self.db.updateContext(self.userId) - logger.debug(f"createAutomationDefinition: Updated database context with userId={self.userId}") - except Exception as e: - logger.warning(f"Could not update database context: {e}") - - # Create automation in database - createdAutomation = self.db.recordCreate(AutomationDefinition, automationData) - - # Compute status - createdAutomation["status"] = self._computeAutomationStatus(createdAutomation) - # Ensure executionLogs is always a list, not None - if createdAutomation.get("executionLogs") is None: - createdAutomation["executionLogs"] = [] - - # Trigger automation change callback - self._notifyAutomationChanged() - - # Clean metadata fields and return Pydantic model - cleanedRecord = _automationDefinitionPayload(createdAutomation) - return AutomationDefinition(**cleanedRecord) - except Exception as e: - logger.error(f"Error creating automation definition: {str(e)}") - raise - - def _saveExecutionLog(self, automationId: str, executionLogs: List[Dict[str, Any]]) -> None: - """ - Save execution logs to an automation definition WITHOUT RBAC check. - - This is a system-level operation: when a user executes an automation, - the execution log must be saved regardless of whether the user has - 'update' permission on the AutomationDefinition. The user already - proved they have execute/read access by loading the automation. - """ - try: - self.db.recordModify(AutomationDefinition, automationId, {"executionLogs": executionLogs}) - logger.debug(f"Saved execution log for automation {automationId}") - except Exception as e: - logger.warning(f"Could not save execution log for automation {automationId}: {e}") - - def updateAutomationDefinition(self, automationId: str, automationData: Dict[str, Any]) -> AutomationDefinition: - """Updates an automation definition, then triggers sync.""" - try: - # Check access - existing = self.getAutomationDefinition(automationId) - if not existing: - raise PermissionError(f"No access to automation {automationId}") - - if not self.checkRbacPermission(AutomationDefinition, "update", automationId): - raise PermissionError(f"No permission to modify automation {automationId}") - - automationData.pop("executionLogs", None) - - # If deactivating: immediately remove scheduler job (don't rely on async callback) - isBeingDeactivated = "active" in automationData and not automationData["active"] - if isBeingDeactivated: - existingEventId = getattr(existing, "eventId", None) if not isinstance(existing, dict) else existing.get("eventId") - if existingEventId: - try: - from modules.shared.eventManagement import eventManager - eventManager.remove(existingEventId) - logger.info(f"Removed scheduler job {existingEventId} (automation deactivated)") - except Exception as e: - logger.warning(f"Could not remove scheduler job {existingEventId}: {e}") - automationData["eventId"] = None - - # Update automation in database - updatedAutomation = self.db.recordModify(AutomationDefinition, automationId, automationData) - - # Compute status - updatedAutomation["status"] = self._computeAutomationStatus(updatedAutomation) - # Ensure executionLogs is always a list, not None - if updatedAutomation.get("executionLogs") is None: - updatedAutomation["executionLogs"] = [] - - # Trigger automation change callback - self._notifyAutomationChanged() - - # Clean metadata fields and return Pydantic model - cleanedRecord = _automationDefinitionPayload(updatedAutomation) - return AutomationDefinition(**cleanedRecord) - except Exception as e: - logger.error(f"Error updating automation definition: {str(e)}") - raise - - def deleteAutomationDefinition(self, automationId: str) -> bool: - """Deletes an automation definition, then triggers sync.""" - try: - # Check access - existing = self.getAutomationDefinition(automationId) - if not existing: - raise PermissionError(f"No access to automation {automationId}") - - if not self.checkRbacPermission(AutomationDefinition, "delete", automationId): - raise PermissionError(f"No permission to delete automation {automationId}") - - # Delete automation from database - self.db.recordDelete(AutomationDefinition, automationId) - - # Trigger automation change callback - self._notifyAutomationChanged() - - return True - except Exception as e: - logger.error(f"Error deleting automation definition: {str(e)}") - raise - - def getAllAutomationDefinitionsWithRBAC(self, user: User) -> List[Dict[str, Any]]: - """ - Get all automation definitions filtered by RBAC for a specific user. - This method encapsulates getRecordsetWithRBAC() to avoid exposing the connector. - - Args: - user: User object for RBAC filtering - - Returns: - List of automation definition dictionaries filtered by RBAC - """ - return getRecordsetWithRBAC( - self.db, - AutomationDefinition, - user, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId - ) - - # ========================================================================= - # AutomationTemplate CRUD methods - # ========================================================================= - - def getAllAutomationTemplates(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: - """ - Returns automation templates: system templates + instance templates for current instance. - System templates (isSystem=True) are always included (read-only for non-SysAdmin). - Instance templates (featureInstanceId matches) are included with RBAC filtering. - """ - # Load ALL templates and filter in Python. - # Reason: seeded/legacy templates may have isSystem=NULL (not False/True), - # which breaks SQL equality filters (NULL != True AND NULL != False). - allTemplates = self.db.getRecordset(AutomationTemplate) - - filteredTemplates = [] - for t in allTemplates: - isSystem = t.get("isSystem") - fid = t.get("featureInstanceId") - - if isSystem is True: - # System templates — always visible to all users - filteredTemplates.append(t) - elif fid and fid == self.featureInstanceId: - # Instance templates — scoped to current feature instance - filteredTemplates.append(t) - elif not fid: - # Global/legacy templates (no featureInstanceId) — visible to all users - filteredTemplates.append(t) - - # Enrich with user names - self._enrichTemplatesWithUserName(filteredTemplates) - - # If no pagination requested, return all items - if pagination is None: - return filteredTemplates - - # Apply filtering (if filters provided) - if pagination.filters: - filteredTemplates = self._applyFilters(filteredTemplates, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination.sort: - filteredTemplates = self._applySorting(filteredTemplates, pagination.sort) - - # Count total items after filters - totalItems = len(filteredTemplates) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedTemplates = filteredTemplates[startIdx:endIdx] - - return PaginatedResult( - items=pagedTemplates, - totalItems=totalItems, - totalPages=totalPages - ) - - def _enrichTemplatesWithUserName(self, templates: List[Dict[str, Any]]) -> None: - """Batch enrich templates with creator user names.""" - if not templates: - return - - # Collect unique user IDs - userIds = set() - for template in templates: - createdBy = template.get("sysCreatedBy") - if createdBy: - userIds.add(createdBy) - - if not userIds: - return - - # Batch fetch users - try: - from modules.datamodels.datamodelUam import UserInDB - from modules.security.rootAccess import getRootDbAppConnector - dbAppConn = getRootDbAppConnector() - - userNameMap = {} - for userId in userIds: - users = dbAppConn.getRecordset(UserInDB, recordFilter={"id": userId}) - if users: - user = users[0] - displayName = user.get("fullName") or user.get("username") or user.get("email") or None - if displayName: - userNameMap[userId] = displayName - - # Apply to templates — SECURITY: no fallback, empty if not found - for template in templates: - createdBy = template.get("sysCreatedBy") - template["sysCreatedByUserName"] = userNameMap.get(createdBy, "") if createdBy else "" - except Exception as e: - logger.warning(f"Could not enrich templates with user names: {e}") - - def getAutomationTemplate(self, templateId: str) -> Optional[Dict[str, Any]]: - """Returns an automation template by ID (system templates always accessible, instance templates scoped).""" - try: - records = self.db.getRecordset( - AutomationTemplate, - recordFilter={"id": templateId} - ) - - if not records: - return None - - template = records[0] - - # System templates are readable by everyone - if template.get("isSystem"): - self._enrichTemplatesWithUserName([template]) - return template - - # Instance templates: must belong to current feature instance - templateInstanceId = template.get("featureInstanceId") - if templateInstanceId and self.featureInstanceId and str(templateInstanceId) != str(self.featureInstanceId): - return None # Not in this instance - - self._enrichTemplatesWithUserName([template]) - return template - except Exception as e: - logger.error(f"Error getting automation template: {str(e)}") - return None - - def createAutomationTemplate(self, templateData: Dict[str, Any], isSysAdmin: bool = False) -> Dict[str, Any]: - """Creates a new automation template. - - System templates (isSystem=True) can only be created by SysAdmin. - Instance templates get featureInstanceId from context. - """ - try: - # Ensure ID is present - if "id" not in templateData or not templateData["id"]: - templateData["id"] = str(uuid.uuid4()) - - # System template protection - if templateData.get("isSystem") and not isSysAdmin: - raise PermissionError("Only SysAdmin can create system templates") - - # Set featureInstanceId for non-system templates - if not templateData.get("isSystem"): - templateData["featureInstanceId"] = self.featureInstanceId - templateData["isSystem"] = False - - # RBAC check (for non-system templates) - if not isSysAdmin and not self.checkRbacPermission(AutomationTemplate, "create"): - raise PermissionError("No permission to create template") - - # Ensure database connector has correct userId context - if self.userId and hasattr(self.db, 'updateContext'): - try: - self.db.updateContext(self.userId) - except Exception as e: - logger.warning(f"Could not update database context: {e}") - - # Convert template field to string if it's a dict (frontend may send parsed JSON) - if "template" in templateData and isinstance(templateData["template"], dict): - import json - templateData["template"] = json.dumps(templateData["template"]) - - # Validate through Pydantic model to ensure proper type conversion - validatedTemplate = AutomationTemplate(**templateData) - - # Create template in database using model_dump for proper serialization - createdTemplate = self.db.recordCreate(AutomationTemplate, validatedTemplate.model_dump()) - - return createdTemplate - except Exception as e: - logger.error(f"Error creating automation template: {str(e)}") - raise - - def updateAutomationTemplate(self, templateId: str, templateData: Dict[str, Any], isSysAdmin: bool = False) -> Dict[str, Any]: - """Updates an automation template. - - System templates can only be updated by SysAdmin. - """ - try: - # Check access - existing = self.getAutomationTemplate(templateId) - if not existing: - raise PermissionError(f"No access to template {templateId}") - - # System template protection - if existing.get("isSystem") and not isSysAdmin: - raise PermissionError("Only SysAdmin can modify system templates") - - if not isSysAdmin and not self.checkRbacPermission(AutomationTemplate, "update", templateId): - raise PermissionError(f"No permission to modify template {templateId}") - - # Prevent changing isSystem/featureInstanceId - templateData.pop("isSystem", None) - templateData.pop("featureInstanceId", None) - - # Convert template field to string if it's a dict (frontend may send parsed JSON) - if "template" in templateData and isinstance(templateData["template"], dict): - import json - templateData["template"] = json.dumps(templateData["template"]) - - # Merge existing data with update data for partial updates - mergedData = {**existing, **templateData} - mergedData["id"] = templateId # Ensure ID is preserved - - # Validate through Pydantic model to ensure proper type conversion - validatedTemplate = AutomationTemplate(**mergedData) - - # Update template in database using model_dump for proper serialization - updatedTemplate = self.db.recordModify(AutomationTemplate, templateId, validatedTemplate.model_dump()) - - return updatedTemplate - except Exception as e: - logger.error(f"Error updating automation template: {str(e)}") - raise - - def deleteAutomationTemplate(self, templateId: str, isSysAdmin: bool = False) -> bool: - """Deletes an automation template. - - System templates can only be deleted by SysAdmin. - """ - try: - # Check access - existing = self.getAutomationTemplate(templateId) - if not existing: - return False - - # System template protection - if existing.get("isSystem") and not isSysAdmin: - raise PermissionError("Only SysAdmin can delete system templates") - - if not isSysAdmin and not self.checkRbacPermission(AutomationTemplate, "delete", templateId): - raise PermissionError(f"No permission to delete template {templateId}") - - # Delete template from database - self.db.recordDelete(AutomationTemplate, templateId) - - return True - except Exception as e: - logger.error(f"Error deleting automation template: {str(e)}") - raise - - def duplicateAutomationTemplate(self, templateId: str) -> Dict[str, Any]: - """Duplicates a template into the current feature instance. - - Creates a copy with new ID, isSystem=False, featureInstanceId from context. - Works for both system and instance templates. - """ - try: - existing = self.getAutomationTemplate(templateId) - if not existing: - raise PermissionError(f"Template {templateId} not found") - - # RBAC check for creating templates - if not self.checkRbacPermission(AutomationTemplate, "create"): - raise PermissionError("No permission to create templates") - - # Build duplicate data - duplicateData = { - "id": str(uuid.uuid4()), - "label": existing.get("label", {}), - "overview": existing.get("overview"), - "template": existing.get("template", ""), - "isSystem": False, - "featureInstanceId": self.featureInstanceId, - } - - # Append "(Kopie)" to label - label = duplicateData["label"] - if isinstance(label, dict): - for lang in label: - if label[lang]: - label[lang] = f"{label[lang]} (Kopie)" - - # Ensure database connector has correct userId context - if self.userId and hasattr(self.db, 'updateContext'): - self.db.updateContext(self.userId) - - validatedTemplate = AutomationTemplate(**duplicateData) - createdTemplate = self.db.recordCreate(AutomationTemplate, validatedTemplate.model_dump()) - - logger.info(f"Duplicated template {templateId} -> {duplicateData['id']}") - return createdTemplate - except Exception as e: - logger.error(f"Error duplicating template: {str(e)}") - raise - - def duplicateAutomationDefinition(self, definitionId: str) -> Dict[str, Any]: - """Duplicates an automation definition within the same feature instance. - - Creates a copy with new ID, active=False, no eventId. - """ - try: - existing = self.getAutomationDefinition(definitionId) - if not existing: - raise PermissionError(f"Definition {definitionId} not found") - - # RBAC check for creating definitions - if not self.checkRbacPermission(AutomationDefinition, "create"): - raise PermissionError("No permission to create definitions") - - # getAutomationDefinition returns Pydantic model; convert to dict for .get() access - existing_data = existing.model_dump() if hasattr(existing, "model_dump") else existing - - # Build duplicate data - duplicateData = { - "id": str(uuid.uuid4()), - "mandateId": existing_data.get("mandateId"), - "featureInstanceId": existing_data.get("featureInstanceId"), - "label": f"{existing_data.get('label', '')} (Kopie)", - "schedule": existing_data.get("schedule", ""), - "template": existing_data.get("template", ""), - "placeholders": existing_data.get("placeholders", {}), - "active": False, - "eventId": None, - "status": None, - "executionLogs": [], - "allowedProviders": existing_data.get("allowedProviders", []), - } - - # Ensure database connector has correct userId context - if self.userId and hasattr(self.db, 'updateContext'): - self.db.updateContext(self.userId) - - validatedDefinition = AutomationDefinition(**duplicateData) - createdDefinition = self.db.recordCreate(AutomationDefinition, validatedDefinition.model_dump()) - - logger.info(f"Duplicated definition {definitionId} -> {duplicateData['id']}") - return createdDefinition - except Exception as e: - logger.error(f"Error duplicating definition: {str(e)}") - raise - - def _notifyAutomationChanged(self): - """Notify registered callbacks about automation changes (decoupled from features). - Sync-safe: works from both sync and async contexts.""" - try: - from modules.shared.callbackRegistry import callbackRegistry - # Trigger callbacks without knowing which features are listening - callbackRegistry.trigger('automation.changed', self) - except Exception as e: - logger.error(f"Error notifying automation change: {str(e)}") - - -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'AutomationObjects': - """ - Returns an AutomationObjects instance for the current user. - Handles initialization of database and records. - - Args: - currentUser: The authenticated user - mandateId: The mandate ID from RequestContext (X-Mandate-Id header). - featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). - """ - if not currentUser: - raise ValueError("Invalid user context: user is required") - - effectiveMandateId = str(mandateId) if mandateId else None - effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - - # Create context key including featureInstanceId for proper isolation - contextKey = f"automation_{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" - - # Create new instance if not exists - if contextKey not in _automationInterfaces: - _automationInterfaces[contextKey] = AutomationObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) - else: - # Update user context if needed - _automationInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) - - return _automationInterfaces[contextKey] diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py deleted file mode 100644 index d56804fd..00000000 --- a/modules/features/automation/mainAutomation.py +++ /dev/null @@ -1,446 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Automation Feature Container - Main Module. -Handles feature initialization and RBAC catalog registration. -""" - -import logging -from typing import Dict, List, Any, Optional - -logger = logging.getLogger(__name__) - -# Feature metadata -FEATURE_CODE = "automation" -FEATURE_LABEL = {"en": "Automation", "de": "Automatisierung", "fr": "Automatisation"} -FEATURE_ICON = "mdi-cog-clockwise" - -# UI Objects for RBAC catalog -UI_OBJECTS = [ - { - "objectKey": "ui.feature.automation.definitions", - "label": {"en": "Automation Definitions", "de": "Automatisierungs-Definitionen", "fr": "Définitions d'automatisation"}, - "meta": {"area": "definitions"} - }, - { - "objectKey": "ui.feature.automation.templates", - "label": {"en": "Templates", "de": "Vorlagen", "fr": "Modèles"}, - "meta": {"area": "templates"} - }, -] - -# Resource Objects for RBAC catalog -RESOURCE_OBJECTS = [ - { - "objectKey": "resource.feature.automation.create", - "label": {"en": "Create Automation", "de": "Automatisierung erstellen", "fr": "Créer automatisation"}, - "meta": {"endpoint": "/api/automations", "method": "POST"} - }, - { - "objectKey": "resource.feature.automation.update", - "label": {"en": "Update Automation", "de": "Automatisierung aktualisieren", "fr": "Modifier automatisation"}, - "meta": {"endpoint": "/api/automations/{automationId}", "method": "PUT"} - }, - { - "objectKey": "resource.feature.automation.delete", - "label": {"en": "Delete Automation", "de": "Automatisierung löschen", "fr": "Supprimer automatisation"}, - "meta": {"endpoint": "/api/automations/{automationId}", "method": "DELETE"} - }, - { - "objectKey": "resource.feature.automation.execute", - "label": {"en": "Execute Automation", "de": "Automatisierung ausführen", "fr": "Exécuter automatisation"}, - "meta": {"endpoint": "/api/automations/{automationId}/execute", "method": "POST"} - }, -] - -# Template roles for this feature -TEMPLATE_ROLES = [ - { - "roleLabel": "automation-admin", - "description": { - "en": "Automation Administrator - Full access to automation configuration and execution", - "de": "Automatisierungs-Administrator - Vollzugriff auf Automatisierungs-Konfiguration und Ausführung", - "fr": "Administrateur automatisation - Accès complet à la configuration et exécution" - }, - "accessRules": [ - # Full UI access - {"context": "UI", "item": None, "view": True}, - # Full DATA access - {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, - ] - }, - { - "roleLabel": "automation-editor", - "description": { - "en": "Automation Editor - Create and modify automations", - "de": "Automatisierungs-Editor - Automatisierungen erstellen und bearbeiten", - "fr": "Éditeur automatisation - Créer et modifier les automatisations" - }, - "accessRules": [ - # UI access to definitions and templates - vollqualifizierte ObjectKeys - {"context": "UI", "item": "ui.feature.automation.definitions", "view": True}, - {"context": "UI", "item": "ui.feature.automation.templates", "view": True}, - {"context": "UI", "item": "ui.feature.automation.logs", "view": True}, - # Group-level DATA access - {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "n"}, - ] - }, - { - "roleLabel": "automation-user", - "description": { - "en": "Automation User - Create and manage own automations", - "de": "Automatisierungs-Benutzer - Eigene Automatisierungen erstellen und verwalten", - "fr": "Utilisateur automatisation - Créer et gérer ses propres automatisations" - }, - "accessRules": [ - {"context": "UI", "item": "ui.feature.automation.definitions", "view": True}, - {"context": "UI", "item": "ui.feature.automation.templates", "view": True}, - {"context": "UI", "item": "ui.feature.automation.logs", "view": True}, - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, - ] - }, - { - "roleLabel": "automation-viewer", - "description": { - "en": "Automation Viewer - View automations and execution results", - "de": "Automatisierungs-Betrachter - Automatisierungen und Ausführungsergebnisse einsehen", - "fr": "Visualiseur automatisation - Consulter les automatisations et résultats" - }, - "accessRules": [ - # UI access to view only - {"context": "UI", "item": "ui.feature.automation.definitions", "view": True}, - {"context": "UI", "item": "ui.feature.automation.logs", "view": True}, - # Read-only DATA access (my level) - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, - ] - }, -] - -# Service requirements - services this feature needs from the service center -REQUIRED_SERVICES = [ - {"serviceKey": "chat", "meta": {"usage": "Workflow CRUD, messages, logs"}}, - {"serviceKey": "ai", "meta": {"usage": "AI planning for workflow execution"}}, - {"serviceKey": "utils", "meta": {"usage": "Timestamps, utilities"}}, - {"serviceKey": "billing", "meta": {"usage": "AI call billing"}}, - {"serviceKey": "extraction", "meta": {"usage": "Workflow method actions"}}, - {"serviceKey": "sharepoint", "meta": {"usage": "SharePoint actions (listDocuments, uploadDocument, etc.)"}}, - {"serviceKey": "generation", "meta": {"usage": "Action completion messages, document creation from results"}}, -] - - -def getRequiredServiceKeys() -> List[str]: - """Return list of service keys this feature requires.""" - return [s["serviceKey"] for s in REQUIRED_SERVICES] - - -def getAutomationServices( - user, - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - workflow=None, -) -> "_AutomationServiceHub": - """ - Get a service hub for the automation feature using the service center. - Resolves only the services declared in REQUIRED_SERVICES. - No legacy fallback - service center only. - - Returns a hub-like object with: chat, ai, utils, billing, extraction, - sharepoint, rbac, interfaceDbApp, interfaceDbComponent, interfaceDbChat, - interfaceDbAutomation. - """ - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - from modules.features.automation.interfaceFeatureAutomation import getInterface as getAutomationInterface - - _workflow = workflow - if _workflow is None: - # Placeholder must have 'id' and 'workflowMode' to avoid AttributeError when services use context.workflow - _workflow = type("_Placeholder", (), {"featureCode": FEATURE_CODE, "id": None, "workflowMode": None})() - ctx = ServiceCenterContext( - user=user, - mandate_id=mandateId, - feature_instance_id=featureInstanceId, - workflow=_workflow, - ) - - hub = _AutomationServiceHub() - hub.user = user - hub.mandateId = mandateId - hub.featureInstanceId = featureInstanceId - hub._service_context = ctx # Store context so workflow updates propagate to services - hub.workflow = workflow - hub.featureCode = FEATURE_CODE - hub.allowedProviders = None - - for spec in REQUIRED_SERVICES: - key = spec["serviceKey"] - try: - svc = getService(key, ctx) - setattr(hub, key, svc) - except Exception as e: - logger.warning(f"Could not resolve service '{key}' for automation: {e}") - setattr(hub, key, None) - - # Copy interfaces from chat service for WorkflowManager compatibility - if hub.chat: - hub.interfaceDbApp = getattr(hub.chat, "interfaceDbApp", None) - hub.interfaceDbComponent = getattr(hub.chat, "interfaceDbComponent", None) - hub.interfaceDbChat = getattr(hub.chat, "interfaceDbChat", None) - - # RBAC for MethodBase action permission checks (workflow methods) - hub.rbac = getattr(hub.interfaceDbApp, "rbac", None) if hub.interfaceDbApp else None - - # Set interfaceDbAutomation from feature interface - hub.interfaceDbAutomation = getAutomationInterface( - user, mandateId=mandateId, featureInstanceId=featureInstanceId - ) - - return hub - - -class _AutomationServiceHub: - """Lightweight hub exposing only services required by the automation feature.""" - - user = None - mandateId = None - featureInstanceId = None - _service_context = None # ServiceCenterContext; when workflow is set, context.workflow is updated - workflow = None - featureCode = "automation" - allowedProviders = None - interfaceDbApp = None - interfaceDbComponent = None - interfaceDbChat = None - interfaceDbAutomation = None - rbac = None - chat = None - ai = None - utils = None - billing = None - extraction = None - sharepoint = None - - -def getFeatureDefinition() -> Dict[str, Any]: - """Return the feature definition for registration.""" - return { - "code": FEATURE_CODE, - "label": FEATURE_LABEL, - "icon": FEATURE_ICON, - "autoCreateInstance": False, - } - - -def getUiObjects() -> List[Dict[str, Any]]: - """Return UI objects for RBAC catalog registration.""" - return UI_OBJECTS - - -def getResourceObjects() -> List[Dict[str, Any]]: - """Return resource objects for RBAC catalog registration.""" - return RESOURCE_OBJECTS - - -def getTemplateRoles() -> List[Dict[str, Any]]: - """Return template roles for this feature.""" - return TEMPLATE_ROLES - - -def registerFeature(catalogService) -> bool: - """ - Register this feature's RBAC objects in the catalog. - - Args: - catalogService: The RBAC catalog service instance - - Returns: - True if registration was successful - """ - try: - # Register UI objects - for uiObj in UI_OBJECTS: - catalogService.registerUiObject( - featureCode=FEATURE_CODE, - objectKey=uiObj["objectKey"], - label=uiObj["label"], - meta=uiObj.get("meta") - ) - - # Register Resource objects - for resObj in RESOURCE_OBJECTS: - catalogService.registerResourceObject( - featureCode=FEATURE_CODE, - objectKey=resObj["objectKey"], - label=resObj["label"], - meta=resObj.get("meta") - ) - - # Sync template roles to database - _syncTemplateRolesToDb() - - # Mark existing templates without isSystem field as system templates (migration) - _migrateExistingTemplates() - - logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") - return True - - except Exception as e: - logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") - return False - - -def _syncTemplateRolesToDb() -> int: - """ - Sync template roles and their AccessRules to the database. - Creates global template roles (mandateId=None) if they don't exist. - - Returns: - Number of roles created/updated - """ - try: - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext - - rootInterface = getRootInterface() - - # Get existing template roles for this feature (Pydantic models) - existingRoles = rootInterface.getRolesByFeatureCode(FEATURE_CODE) - # Filter to template roles (mandateId is None) - templateRoles = [r for r in existingRoles if r.mandateId is None] - existingRoleLabels = {r.roleLabel: str(r.id) for r in templateRoles} - - createdCount = 0 - for roleTemplate in TEMPLATE_ROLES: - roleLabel = roleTemplate["roleLabel"] - - if roleLabel in existingRoleLabels: - roleId = existingRoleLabels[roleLabel] - # Ensure AccessRules exist for this role - _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) - else: - # Create new template role - newRole = Role( - roleLabel=roleLabel, - description=roleTemplate.get("description", {}), - featureCode=FEATURE_CODE, - mandateId=None, # Global template - featureInstanceId=None, - isSystemRole=False - ) - createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) - roleId = createdRole.get("id") - - # Create AccessRules for this role - _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) - - logger.info(f"Created template role '{roleLabel}' with ID {roleId}") - createdCount += 1 - - if createdCount > 0: - logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") - - return createdCount - - except Exception as e: - logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") - return 0 - - -def _ensureAccessRulesForRole(rootInterface, roleId: str, ruleTemplates: List[Dict[str, Any]]) -> int: - """ - Ensure AccessRules exist for a role based on templates. - - Args: - rootInterface: Root interface instance - roleId: Role ID - ruleTemplates: List of rule templates - - Returns: - Number of rules created - """ - from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext - - # Get existing rules for this role (Pydantic models) - existingRules = rootInterface.getAccessRulesByRole(roleId) - - # Create a set of existing rule signatures to avoid duplicates - # IMPORTANT: Use .value for enum comparison, not str() which gives "AccessRuleContext.DATA" in Python 3.11+ - existingSignatures = set() - for rule in existingRules: - sig = (rule.context.value if rule.context else None, rule.item) - existingSignatures.add(sig) - - createdCount = 0 - for template in ruleTemplates: - context = template.get("context", "UI") - item = template.get("item") - sig = (context, item) - - if sig in existingSignatures: - continue - - # Map context string to enum - if context == "UI": - contextEnum = AccessRuleContext.UI - elif context == "DATA": - contextEnum = AccessRuleContext.DATA - elif context == "RESOURCE": - contextEnum = AccessRuleContext.RESOURCE - else: - contextEnum = context - - newRule = AccessRule( - roleId=roleId, - context=contextEnum, - item=item, - view=template.get("view", False), - read=template.get("read"), - create=template.get("create"), - update=template.get("update"), - delete=template.get("delete"), - ) - rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) - createdCount += 1 - - if createdCount > 0: - logger.debug(f"Created {createdCount} AccessRules for role {roleId}") - - return createdCount - - -def _migrateExistingTemplates() -> None: - """ - Migration: Mark existing templates that have no isSystem/featureInstanceId fields - as system templates (isSystem=True). This runs idempotently during feature registration. - """ - try: - from modules.features.automation.interfaceFeatureAutomation import getInterface - from modules.security.rootAccess import getRootUser - from modules.features.automation.datamodelFeatureAutomation import AutomationTemplate - - rootUser = getRootUser() - automationInterface = getInterface(rootUser) - - # Get all templates from DB - allTemplates = automationInterface.db.getRecordset(AutomationTemplate) - - migratedCount = 0 - for template in allTemplates: - templateId = template.get("id") - isSystem = template.get("isSystem") - featureInstanceId = template.get("featureInstanceId") - - # Templates without isSystem set (old templates) → mark as system - if isSystem is None and featureInstanceId is None: - automationInterface.db.recordModify( - AutomationTemplate, - templateId, - {"isSystem": True, "featureInstanceId": None} - ) - migratedCount += 1 - - if migratedCount > 0: - logger.info(f"Migrated {migratedCount} existing templates to isSystem=True") - - except Exception as e: - logger.warning(f"Template migration check failed (non-critical): {e}") diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py deleted file mode 100644 index c6343b25..00000000 --- a/modules/features/automation/routeFeatureAutomation.py +++ /dev/null @@ -1,1264 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Automation routes for the backend API. -Implements the endpoints for automation definition management. -""" - -from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response, Query -from typing import List, Dict, Any, Optional -from fastapi import status -from fastapi.responses import JSONResponse -import logging -import json - -# Import interfaces and models -from modules.features.automation.interfaceFeatureAutomation import getInterface as getAutomationInterface -from modules.features.automation.mainAutomation import getAutomationServices -from modules.auth import limiter, getRequestContext, RequestContext -from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition, AutomationTemplate -from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog, UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.shared.attributeUtils import getModelAttributeDefinitions -from modules.interfaces import interfaceDbChat -from modules.interfaces.interfaceDbBilling import getInterface as _getBillingInterface -# Configure logger -logger = logging.getLogger(__name__) - -# Model attributes for AutomationDefinition and ChatWorkflow -automationAttributes = getModelAttributeDefinitions(AutomationDefinition) -workflowAttributes = getModelAttributeDefinitions(ChatWorkflow) - -# Create router for automation endpoints -router = APIRouter( - prefix="/api/automations", - tags=["Manage Automations"], - responses={ - 404: {"description": "Not found"}, - 400: {"description": "Bad request"}, - 401: {"description": "Unauthorized"}, - 403: {"description": "Forbidden"}, - 500: {"description": "Internal server error"} - } -) - -@router.get("", response_model=PaginatedResponse[AutomationDefinition]) -@limiter.limit("30/minute") -def get_automations( - request: Request, - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - context: RequestContext = Depends(getRequestContext) -) -> PaginatedResponse[AutomationDefinition]: - """ - Get automation definitions with optional pagination, sorting, and filtering. - - Query Parameters: - - pagination: JSON-encoded PaginationParams object, or None for no pagination - """ - try: - # Parse pagination parameter - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException( - status_code=400, - detail=f"Invalid pagination parameter: {str(e)}" - ) - - # AutomationDefinitions can belong to ANY feature instance within a mandate. - # The list endpoint must show all definitions for the user's mandate, not filter by a specific featureInstanceId. - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - result = chatInterface.getAllAutomationDefinitions(pagination=paginationParams) - - # If pagination was requested, result is PaginatedResult - # If no pagination, result is List[Dict] - # Note: Using JSONResponse to bypass Pydantic validation which would filter out sysCreatedBy - # The enriched fields (sysCreatedByUserName, mandateName) are not in the Pydantic model - from fastapi.responses import JSONResponse - - if paginationParams: - response_data = { - "items": result.items, - "pagination": { - "currentPage": paginationParams.page, - "pageSize": paginationParams.pageSize, - "totalItems": result.totalItems, - "totalPages": result.totalPages, - "sort": paginationParams.sort, - "filters": paginationParams.filters - } - } - else: - response_data = { - "items": result, - "pagination": None - } - - return JSONResponse(content=response_data) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting automations: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting automations: {str(e)}" - ) - -@router.get("/filter-values") -@limiter.limit("60/minute") -def get_automation_filter_values( - request: Request, - column: str = Query(..., description="Column key"), - pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), - context: RequestContext = Depends(getRequestContext) -) -> list: - """Return distinct filter values for a column in automations.""" - try: - from modules.routes.routeDataUsers import _handleFilterValuesRequest - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - result = chatInterface.getAllAutomationDefinitions(pagination=None) - items = result if isinstance(result, list) else [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result] - return _handleFilterValuesRequest(items, column, pagination) - except Exception as e: - logger.error(f"Error getting filter values for automations: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("", response_model=AutomationDefinition) -@limiter.limit("10/minute") -def create_automation( - request: Request, - automation: AutomationDefinition, - context: RequestContext = Depends(getRequestContext) -) -> AutomationDefinition: - """Create a new automation definition""" - try: - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) - automationData = automation.model_dump() - created = chatInterface.createAutomationDefinition(automationData) - return created - except HTTPException: - raise - except Exception as e: - logger.error(f"Error creating automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error creating automation: {str(e)}" - ) - -@router.get("/attributes", response_model=Dict[str, Any]) -def get_automation_attributes( - request: Request -) -> Dict[str, Any]: - """Get attribute definitions for AutomationDefinition model""" - return {"attributes": automationAttributes} - - -@router.get("/actions") -@limiter.limit("30/minute") -def get_available_actions( - request: Request, - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """ - Get available workflow actions for template editor. - Returns action definitions with parameters and example JSON snippets. - """ - try: - from modules.workflows.processing.shared.methodDiscovery import methods, discoverMethods - - # Ensure methods are discovered (need a service hub for discovery) - if not methods: - services = getAutomationServices( - context.user, - mandateId=context.mandateId, - featureInstanceId=context.featureInstanceId, - ) - discoverMethods(services) - - actionsList = [] - processedMethods = set() - - for methodName, methodInfo in methods.items(): - # Skip short name aliases - only process full class names (MethodXxx) - if not methodName.startswith('Method'): - continue - - shortName = methodName.replace('Method', '').lower() - - # Skip if already processed - if shortName in processedMethods: - continue - processedMethods.add(shortName) - - methodInstance = methodInfo.get('instance') - if not methodInstance: - continue - - # Get actions from method instance - for actionName, actionDef in methodInstance._actions.items(): - # Build action info - actionInfo = { - "method": shortName, - "action": actionName, - "actionId": actionDef.actionId if hasattr(actionDef, 'actionId') else f"{shortName}.{actionName}", - "description": actionDef.description if hasattr(actionDef, 'description') else "", - "category": actionDef.category if hasattr(actionDef, 'category') else "general", - "parameters": [] - } - - # Add parameters from WorkflowActionParameter - parametersDef = actionDef.parameters if hasattr(actionDef, 'parameters') else {} - for paramName, paramDef in parametersDef.items(): - paramInfo = { - "name": paramName, - "type": paramDef.type if hasattr(paramDef, 'type') else "Any", - "frontendType": paramDef.frontendType.value if hasattr(paramDef, 'frontendType') and paramDef.frontendType else "text", - "required": paramDef.required if hasattr(paramDef, 'required') else False, - "default": paramDef.default if hasattr(paramDef, 'default') else None, - "description": paramDef.description if hasattr(paramDef, 'description') else "", - } - if hasattr(paramDef, 'frontendOptions') and paramDef.frontendOptions: - paramInfo["frontendOptions"] = paramDef.frontendOptions - actionInfo["parameters"].append(paramInfo) - - # Build example JSON snippet for copy/paste - exampleParams = {} - for paramName, paramDef in parametersDef.items(): - if hasattr(paramDef, 'required') and paramDef.required: - exampleParams[paramName] = f"{{{{KEY:{paramName}}}}}" - else: - default = paramDef.default if hasattr(paramDef, 'default') else None - exampleParams[paramName] = default or f"{{{{KEY:{paramName}}}}}" - - actionInfo["exampleJson"] = { - "execMethod": shortName, - "execAction": actionName, - "execParameters": exampleParams, - "execResultLabel": f"{shortName}_{actionName}_result" - } - - actionsList.append(actionInfo) - - return JSONResponse(content={"actions": actionsList}) - except Exception as e: - logger.error(f"Error getting available actions: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting available actions: {str(e)}" - ) - - -# ----------------------------------------------------------------------------- -# Workflow routes under /{instanceId}/workflows/ (instance-scoped) -# ----------------------------------------------------------------------------- - -def _validateAutomationInstanceAccess(instanceId: str, context: RequestContext) -> Optional[str]: - """Validate user has access to the automation feature instance. Returns mandateId.""" - from modules.interfaces.interfaceDbApp import getRootInterface - rootInterface = getRootInterface() - instance = rootInterface.getFeatureInstance(instanceId) - if not instance: - raise HTTPException(status_code=404, detail=f"Feature instance {instanceId} not found") - featureAccess = rootInterface.getFeatureAccess(str(context.user.id), instanceId) - if not featureAccess or not featureAccess.enabled: - raise HTTPException(status_code=403, detail="Access denied to this feature instance") - return str(instance.mandateId) if instance.mandateId else None - - -def _getAutomationServiceChat(context: RequestContext, featureInstanceId: str = None, mandateId: str = None): - """Get chat interface with feature instance context for workflows.""" - return interfaceDbChat.getInterface( - context.user, - mandateId=mandateId or (str(context.mandateId) if context.mandateId else None), - featureInstanceId=featureInstanceId - ) - - -@router.get("/{instanceId}/workflows", response_model=PaginatedResponse[ChatWorkflow]) -@limiter.limit("120/minute") -def get_automation_workflows( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - page: int = Query(1, ge=1, description="Page number (legacy)"), - pageSize: int = Query(20, ge=1, le=100, description="Items per page (legacy)"), - context: RequestContext = Depends(getRequestContext) -) -> PaginatedResponse[ChatWorkflow]: - """Get all workflows for this automation feature instance.""" - try: - mandateId = _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId, mandateId=mandateId) - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") - else: - paginationParams = PaginationParams(page=page, pageSize=pageSize) - result = chatInterface.getWorkflows(pagination=paginationParams) - if paginationParams: - return PaginatedResponse( - items=result.items, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=result.totalItems, - totalPages=result.totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters - ) - ) - return PaginatedResponse(items=result, pagination=None) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting automation workflows: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error getting workflows: {str(e)}") - - -# Workflow attributes (ChatWorkflow model) -@router.get("/{instanceId}/workflows/attributes", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -def get_automation_workflow_attributes( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Get attribute definitions for ChatWorkflow model.""" - _validateAutomationInstanceAccess(instanceId, context) - return {"attributes": workflowAttributes} - - -# Actions (must be before /{workflowId} to avoid path conflict) -@router.get("/{instanceId}/workflows/actions", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -def get_automation_workflow_actions( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Get all available workflow actions.""" - try: - mandateId = _validateAutomationInstanceAccess(instanceId, context) - services = getAutomationServices(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods - discoverMethods(services) - allActions = [] - for methodName, methodInfo in methods.items(): - if methodName.startswith('Method'): - continue - methodInstance = methodInfo['instance'] - for actionName, actionInfo in methodInstance.actions.items(): - allActions.append({ - "module": methodInstance.name, - "actionId": f"{methodInstance.name}.{actionName}", - "name": actionName, - "description": actionInfo.get('description', ''), - "parameters": actionInfo.get('parameters', {}) - }) - return {"actions": allActions} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting actions: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get actions: {str(e)}") - - -@router.get("/{instanceId}/workflows/actions/{method}", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -def get_automation_method_actions( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - method: str = Path(..., description="Method name"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Get actions for a specific method.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - services = getAutomationServices(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=instanceId) - from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods - discoverMethods(services) - methodInstance = None - for mn, mi in methods.items(): - if mi['instance'].name == method: - methodInstance = mi['instance'] - break - if not methodInstance: - raise HTTPException(status_code=404, detail=f"Method '{method}' not found") - actions = [{"actionId": f"{methodInstance.name}.{an}", "name": an, "description": ai.get('description', ''), "parameters": ai.get('parameters', {})} - for an, ai in methodInstance.actions.items()] - return {"module": methodInstance.name, "description": methodInstance.description, "actions": actions} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting actions for {method}: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get actions: {str(e)}") - - -@router.get("/{instanceId}/workflows/actions/{method}/{action}", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -def get_automation_action_schema( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - method: str = Path(..., description="Method name"), - action: str = Path(..., description="Action name"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Get action schema for a specific action.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - services = getAutomationServices(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=instanceId) - from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods - discoverMethods(services) - methodInstance = None - for mn, mi in methods.items(): - if mi['instance'].name == method: - methodInstance = mi['instance'] - break - if not methodInstance: - raise HTTPException(status_code=404, detail=f"Method '{method}' not found") - if action not in methodInstance.actions: - raise HTTPException(status_code=404, detail=f"Action '{action}' not found in method '{method}'") - ai = methodInstance.actions[action] - return {"method": methodInstance.name, "action": action, "actionId": f"{methodInstance.name}.{action}", - "description": ai.get('description', ''), "parameters": ai.get('parameters', {})} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting action schema: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to get action schema: {str(e)}") - - -@router.get("/{instanceId}/workflows/{workflowId}", response_model=ChatWorkflow) -@limiter.limit("120/minute") -def get_automation_workflow( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Get workflow by ID.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail="Workflow not found") - return workflow - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting workflow: {str(e)}") - raise HTTPException(status_code=500, detail=f"Failed to get workflow: {str(e)}") - - -@router.put("/{instanceId}/workflows/{workflowId}", response_model=ChatWorkflow) -@limiter.limit("120/minute") -def update_automation_workflow( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - workflowData: Dict[str, Any] = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Update workflow by ID.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail="Workflow not found") - if not chatInterface.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise HTTPException(status_code=403, detail="You don't have permission to update this workflow") - updated = chatInterface.updateWorkflow(workflowId, workflowData) - if not updated: - raise HTTPException(status_code=500, detail="Failed to update workflow") - return updated - except HTTPException: - raise - except Exception as e: - logger.error(f"Error updating workflow: {str(e)}") - raise HTTPException(status_code=500, detail=f"Failed to update workflow: {str(e)}") - - -@router.delete("/{instanceId}/workflows/{workflowId}") -@limiter.limit("120/minute") -def delete_automation_workflow( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Delete workflow and associated data.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - if not chatInterface.checkRbacPermission(ChatWorkflow, "delete", workflowId): - raise HTTPException(status_code=403, detail="You don't have permission to delete this workflow") - success = chatInterface.deleteWorkflow(workflowId) - if not success: - raise HTTPException(status_code=500, detail="Failed to delete workflow") - return {"id": workflowId, "message": "Workflow and associated data deleted successfully"} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error deleting workflow: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error deleting workflow: {str(e)}") - - -@router.get("/{instanceId}/workflows/{workflowId}/status", response_model=ChatWorkflow) -@limiter.limit("120/minute") -def get_automation_workflow_status( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Get workflow status.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - return workflow - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting workflow status: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error getting workflow status: {str(e)}") - - -@router.get("/{instanceId}/workflows/{workflowId}/logs", response_model=PaginatedResponse[ChatLog]) -@limiter.limit("120/minute") -def get_automation_workflow_logs( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - logId: Optional[str] = Query(None), - pagination: Optional[str] = Query(None), - context: RequestContext = Depends(getRequestContext) -) -> PaginatedResponse[ChatLog]: - """Get workflow logs.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - paginationParams = None - if pagination: - try: - pd = json.loads(pagination) - if pd: - pd = normalize_pagination_dict(pd) - paginationParams = PaginationParams(**pd) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination: {str(e)}") - result = chatInterface.getLogs(workflowId, pagination=paginationParams) - if logId: - allLogs = result.items if paginationParams else result - idx = next((i for i, log in enumerate(allLogs) if log.id == logId), -1) - if idx >= 0: - return PaginatedResponse(items=allLogs[idx + 1:], pagination=None) - if paginationParams: - return PaginatedResponse(items=result.items, pagination=PaginationMetadata( - currentPage=paginationParams.page, pageSize=paginationParams.pageSize, - totalItems=result.totalItems, totalPages=result.totalPages, - sort=paginationParams.sort, filters=paginationParams.filters)) - return PaginatedResponse(items=result, pagination=None) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting workflow logs: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error getting workflow logs: {str(e)}") - - -@router.get("/{instanceId}/workflows/{workflowId}/messages", response_model=PaginatedResponse[ChatMessage]) -@limiter.limit("120/minute") -def get_automation_workflow_messages( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - messageId: Optional[str] = Query(None), - pagination: Optional[str] = Query(None), - context: RequestContext = Depends(getRequestContext) -) -> PaginatedResponse[ChatMessage]: - """Get workflow messages.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - paginationParams = None - if pagination: - try: - pd = json.loads(pagination) - if pd: - pd = normalize_pagination_dict(pd) - paginationParams = PaginationParams(**pd) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination: {str(e)}") - result = chatInterface.getMessages(workflowId, pagination=paginationParams) - if messageId: - allMsgs = result.items if paginationParams else result - idx = next((i for i, m in enumerate(allMsgs) if m.id == messageId), -1) - if idx >= 0: - return PaginatedResponse(items=allMsgs[idx + 1:], pagination=None) - if paginationParams: - return PaginatedResponse(items=result.items, pagination=PaginationMetadata( - currentPage=paginationParams.page, pageSize=paginationParams.pageSize, - totalItems=result.totalItems, totalPages=result.totalPages, - sort=paginationParams.sort, filters=paginationParams.filters)) - return PaginatedResponse(items=result, pagination=None) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting workflow messages: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error getting workflow messages: {str(e)}") - - -@router.delete("/{instanceId}/workflows/{workflowId}/messages/{messageId}") -@limiter.limit("120/minute") -def delete_automation_workflow_message( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - messageId: str = Path(..., description="Message ID"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Delete message from workflow.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - success = chatInterface.deleteMessage(workflowId, messageId) - if not success: - raise HTTPException(status_code=404, detail=f"Message {messageId} not found") - return {"workflowId": workflowId, "messageId": messageId, "message": "Message deleted successfully"} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error deleting message: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error deleting message: {str(e)}") - - -@router.delete("/{instanceId}/workflows/{workflowId}/messages/{messageId}/files/{fileId}") -@limiter.limit("120/minute") -def delete_automation_file_from_message( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - messageId: str = Path(..., description="Message ID"), - fileId: str = Path(..., description="File ID"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Delete file from message.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - success = chatInterface.deleteFileFromMessage(workflowId, messageId, fileId) - if not success: - raise HTTPException(status_code=404, detail=f"File {fileId} not found") - return {"workflowId": workflowId, "messageId": messageId, "fileId": fileId, "message": "File reference deleted successfully"} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error deleting file: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error deleting file: {str(e)}") - - -@router.get("/{instanceId}/workflows/{workflowId}/chatData") -@limiter.limit("120/minute") -def get_automation_workflow_chat_data( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - afterTimestamp: Optional[float] = Query(None), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Get unified chat data for workflow.""" - try: - _validateAutomationInstanceAccess(instanceId, context) - chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId) - workflow = chatInterface.getWorkflow(workflowId) - if not workflow: - raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - billingInterface = _getBillingInterface(context.user, context.mandateId) - workflowCost = billingInterface.getWorkflowCost(workflowId) - return chatInterface.getUnifiedChatData(workflowId, afterTimestamp, workflowCost=workflowCost) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting chat data: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error getting chat data: {str(e)}") - - -@router.post("/{instanceId}/workflows/{workflowId}/stop", response_model=ChatWorkflow) -@limiter.limit("30/minute") -async def stop_automation_workflow( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: str = Path(..., description="Workflow ID"), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Stop a running automation workflow. Uses instance-scoped services.""" - try: - from modules.workflows.automation import chatStop - mandateId = _validateAutomationInstanceAccess(instanceId, context) - services = getAutomationServices( - context.user, - mandateId=mandateId, - featureInstanceId=instanceId, - ) - services.featureCode = "automation" - return await chatStop( - context.user, - workflowId, - mandateId=mandateId, - featureInstanceId=instanceId, - featureCode="automation", - services=services, - ) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error stopping automation workflow: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.get("/{automationId}", response_model=AutomationDefinition) -@limiter.limit("30/minute") -def get_automation( - request: Request, - automationId: str = Path(..., description="Automation ID"), - context: RequestContext = Depends(getRequestContext) -) -> AutomationDefinition: - """Get a single automation definition by ID""" - try: - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - automation = chatInterface.getAutomationDefinition(automationId) - if not automation: - raise HTTPException( - status_code=404, - detail=f"Automation {automationId} not found" - ) - - return automation - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting automation: {str(e)}" - ) - -@router.put("/{automationId}", response_model=AutomationDefinition) -@limiter.limit("10/minute") -def update_automation( - request: Request, - automationId: str = Path(..., description="Automation ID"), - automation: AutomationDefinition = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> AutomationDefinition: - """Update an automation definition""" - try: - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - automationData = automation.model_dump() - automationData.pop("executionLogs", None) - updated = chatInterface.updateAutomationDefinition(automationId, automationData) - return updated - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error updating automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error updating automation: {str(e)}" - ) - -@router.patch("/{automationId}/status") -@limiter.limit("30/minute") -def update_automation_status( - request: Request, - automationId: str = Path(..., description="Automation ID"), - active: bool = Body(..., embed=True), - context: RequestContext = Depends(getRequestContext) -) -> AutomationDefinition: - """Update only the active status of an automation definition""" - try: - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - - # Get existing automation - automation = chatInterface.getAutomationDefinition(automationId) - if not automation: - raise HTTPException( - status_code=404, - detail=f"Automation {automationId} not found" - ) - - # Update only the active field - automationData = automation if isinstance(automation, dict) else automation.model_dump() - automationData['active'] = active - - updated = chatInterface.updateAutomationDefinition(automationId, automationData) - return updated - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error updating automation status: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error updating automation status: {str(e)}" - ) - - -@router.delete("/{automationId}") -@limiter.limit("10/minute") -def delete_automation( - request: Request, - automationId: str = Path(..., description="Automation ID"), - context: RequestContext = Depends(getRequestContext) -) -> Response: - """Delete an automation definition""" - try: - chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - success = chatInterface.deleteAutomationDefinition(automationId) - if success: - return Response(status_code=204) - else: - raise HTTPException( - status_code=500, - detail="Failed to delete automation" - ) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error deleting automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error deleting automation: {str(e)}" - ) - -@router.post("/{instanceId}/start", response_model=ChatWorkflow) -@limiter.limit("120/minute") -async def start_automation_workflow( - request: Request, - instanceId: str = Path(..., description="Feature instance ID"), - workflowId: Optional[str] = Query(None, description="Optional ID of the workflow to continue"), - workflowMode: WorkflowModeEnum = Query(..., description="Workflow mode: 'Dynamic' or 'Automation' (mandatory)"), - userInput: UserInputRequest = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Start a new workflow or continue an existing one.""" - try: - from modules.workflows.automation import chatStart - mandateId = _validateAutomationInstanceAccess(instanceId, context) - services = getAutomationServices( - context.user, - mandateId=mandateId, - featureInstanceId=instanceId, - ) - services.featureCode = "automation" - if hasattr(userInput, 'allowedProviders') and userInput.allowedProviders: - services.allowedProviders = userInput.allowedProviders - workflow = await chatStart( - context.user, - userInput, - workflowMode, - workflowId, - mandateId=mandateId, - featureInstanceId=instanceId, - featureCode="automation", - services=services, - ) - return workflow - except HTTPException: - raise - except Exception as e: - logger.error(f"Error in start_automation_workflow: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/{automationId}/execute", response_model=ChatWorkflow) -@limiter.limit("5/minute") -async def execute_automation_route( - request: Request, - automationId: str = Path(..., description="Automation ID"), - context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: - """Execute an automation immediately (test mode)""" - try: - services = getAutomationServices( - context.user, - mandateId=context.mandateId, - featureInstanceId=context.featureInstanceId, - ) - - # Load automation with current user's context (user has RBAC permissions via UI) - automation = services.interfaceDbAutomation.getAutomationDefinition(automationId, includeSystemFields=True) - if not automation: - raise ValueError(f"Automation {automationId} not found") - - from modules.workflows.automation import executeAutomation, chatStop - workflow = await executeAutomation(automationId, automation, context.user, services) - return workflow - except HTTPException: - raise - except ValueError as e: - raise HTTPException( - status_code=404, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error executing automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error executing automation: {str(e)}" - ) - - -# ============================================================================= -# AutomationTemplate Routes (DB-persistiert) -# ============================================================================= -# Separater Router für /api/automation-templates - -templateRouter = APIRouter( - prefix="/api/automation-templates", - tags=["Manage Automation Templates"], - responses={ - 404: {"description": "Not found"}, - 400: {"description": "Bad request"}, - 401: {"description": "Unauthorized"}, - 403: {"description": "Forbidden"}, - 500: {"description": "Internal server error"} - } -) - -# Model attributes for AutomationTemplate -templateAttributes = getModelAttributeDefinitions(AutomationTemplate) - - -@templateRouter.get("", response_model=PaginatedResponse[AutomationTemplate]) -@limiter.limit("30/minute") -def get_db_templates( - request: Request, - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """ - Get automation templates from database (RBAC-filtered, MY = own templates). - - Query Parameters: - - pagination: JSON-encoded PaginationParams object, or None for no pagination - """ - try: - # Parse pagination parameter - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException( - status_code=400, - detail=f"Invalid pagination parameter: {str(e)}" - ) - - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - result = chatInterface.getAllAutomationTemplates(pagination=paginationParams) - - if paginationParams: - response_data = { - "items": result.items, - "pagination": { - "currentPage": paginationParams.page, - "pageSize": paginationParams.pageSize, - "totalItems": result.totalItems, - "totalPages": result.totalPages, - "sort": paginationParams.sort, - "filters": paginationParams.filters - } - } - else: - response_data = { - "items": result, - "pagination": None - } - - return JSONResponse(content=response_data) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting templates: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting templates: {str(e)}" - ) - - -@templateRouter.get("/filter-values") -@limiter.limit("60/minute") -def get_template_filter_values( - request: Request, - column: str = Query(..., description="Column key"), - pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), - context: RequestContext = Depends(getRequestContext) -) -> list: - """Return distinct filter values for a column in automation templates.""" - try: - from modules.routes.routeDataUsers import _handleFilterValuesRequest - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - result = chatInterface.getAllAutomationTemplates(pagination=None) - items = [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result] - return _handleFilterValuesRequest(items, column, pagination) - except Exception as e: - logger.error(f"Error getting filter values for automation templates: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - - -@templateRouter.get("/attributes", response_model=Dict[str, Any]) -def get_template_attributes( - request: Request -) -> Dict[str, Any]: - """Get attribute definitions for AutomationTemplate model""" - return {"attributes": templateAttributes} - - -@templateRouter.get("/{templateId}") -@limiter.limit("30/minute") -def get_db_template( - request: Request, - templateId: str = Path(..., description="Template ID"), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """Get a single automation template by ID""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - template = chatInterface.getAutomationTemplate(templateId) - if not template: - raise HTTPException( - status_code=404, - detail=f"Template {templateId} not found" - ) - - return JSONResponse(content=template) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting template: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting template: {str(e)}" - ) - - -@templateRouter.post("") -@limiter.limit("10/minute") -def create_db_template( - request: Request, - templateData: Dict[str, Any] = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """Create a new automation template""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - created = chatInterface.createAutomationTemplate(templateData, isSysAdmin=context.hasSysAdminRole) - return JSONResponse(content=created) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error creating template: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error creating template: {str(e)}" - ) - - -@templateRouter.put("/{templateId}") -@limiter.limit("10/minute") -def update_db_template( - request: Request, - templateId: str = Path(..., description="Template ID"), - templateData: Dict[str, Any] = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """Update an automation template""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - updated = chatInterface.updateAutomationTemplate(templateId, templateData, isSysAdmin=context.hasSysAdminRole) - return JSONResponse(content=updated) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error updating template: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error updating template: {str(e)}" - ) - - -@templateRouter.delete("/{templateId}") -@limiter.limit("10/minute") -def delete_db_template( - request: Request, - templateId: str = Path(..., description="Template ID"), - context: RequestContext = Depends(getRequestContext) -) -> Response: - """Delete an automation template""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - success = chatInterface.deleteAutomationTemplate(templateId, isSysAdmin=context.hasSysAdminRole) - if success: - return Response(status_code=204) - else: - raise HTTPException( - status_code=404, - detail="Template not found or no permission" - ) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error deleting template: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error deleting template: {str(e)}" - ) - - -@templateRouter.post("/{templateId}/duplicate") -@limiter.limit("10/minute") -def duplicate_db_template( - request: Request, - templateId: str = Path(..., description="Template ID to duplicate"), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """Duplicate a template into the current feature instance (system or instance template).""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - duplicated = chatInterface.duplicateAutomationTemplate(templateId) - return JSONResponse(content=duplicated) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error duplicating template: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error duplicating template: {str(e)}" - ) - - -@router.post("/{automationId}/duplicate") -@limiter.limit("10/minute") -def duplicate_automation( - request: Request, - automationId: str = Path(..., description="Automation definition ID to duplicate"), - context: RequestContext = Depends(getRequestContext) -) -> JSONResponse: - """Duplicate an automation definition within the same feature instance.""" - try: - chatInterface = getAutomationInterface( - context.user, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - duplicated = chatInterface.duplicateAutomationDefinition(automationId) - return JSONResponse(content=duplicated) - except HTTPException: - raise - except PermissionError as e: - raise HTTPException( - status_code=403, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error duplicating automation: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error duplicating automation: {str(e)}" - ) diff --git a/modules/features/automation/subAutomationTemplates.py b/modules/features/automation/subAutomationTemplates.py deleted file mode 100644 index 0795f757..00000000 --- a/modules/features/automation/subAutomationTemplates.py +++ /dev/null @@ -1,433 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Automation templates for workflow definitions. - -Contains predefined workflow templates that can be used to create automation definitions. -""" - -from typing import Dict, Any, List - -# Automation templates structure -AUTOMATION_TEMPLATES: Dict[str, Any] = { - "sets": [ - { - "template": { - "overview": "SharePoint Themen Zusammenfassung", - "tasks": [ - { - "id": "Task01", - "title": "SharePoint Themen Zusammenfassung", - "description": "Erstellt eine Zusammenfassung aller SharePoint Sites und deren Inhalte", - "objective": "Erstelle eine Zusammenfassung aller SharePoint Themen (Sites) und deren Inhalte als Word-Dokument", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findDocumentPath", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "searchQuery": "*", - "maxResults": 100 - }, - "execResultLabel": "sharepoint_sites_found" - }, - { - "execMethod": "sharepoint", - "execAction": "listDocuments", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "pathQuery": "{{KEY:sharepointBasePath}}", - "includeSubfolders": True - }, - "execResultLabel": "sharepoint_structure" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:summaryPrompt}}", - "documentList": ["sharepoint_sites_found", "sharepoint_structure"], - "resultType": "docx" - }, - "execResultLabel": "sharepoint_summary" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["sharepoint_summary"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "sharepoint_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointBasePath": "/sites/company-share", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "summaryPrompt": "Erstelle eine umfassende Zusammenfassung aller SharePoint Sites und deren Inhalte. Strukturiere das Dokument nach Sites und fasse für jede Site die wichtigsten Themen, Ordnerstrukturen und Dokumente zusammen. Erstelle ein professionelles Word-Dokument mit Überschriften, Abschnitten und einer klaren Gliederung. Berücksichtige alle gefundenen Sites, deren Ordnerstrukturen und dokumentiere die wichtigsten Inhalte pro Site." - } - }, - { - "template": { - "overview": "Immobilienrecherche Zürich", - "tasks": [ - { - "id": "Task02", - "title": "Immobilienrecherche Zürich", - "description": "Webrecherche nach Immobilien im Kanton Zürich und Speicherung in Excel", - "objective": "Immobilienrecherche im Kanton Zürich zum Verkauf (5-20 Mio. CHF) und speichere Ergebnisse in Excel-Liste auf SharePoint", - "actionList": [ - { - "execMethod": "ai", - "execAction": "webResearch", - "execParameters": { - "prompt": "{{KEY:immobilienResearchPrompt}}", - "urlList": ["{{KEY:immobilienResearchUrl}}"] - }, - "execResultLabel": "immobilien_research_results" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:excelFormatPrompt}}", - "documentList": ["immobilien_research_results"], - "resultType": "xlsx" - }, - "execResultLabel": "immobilien_excel_list" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["immobilien_excel_list"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "immobilien_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "immobilienResearchUrl": ["https://www.homegate.ch", "https://www.immoscout24.ch", "https://www.immowelt.ch"], - "immobilienResearchPrompt": "Suche nach Immobilien zum Verkauf im Kanton Zürich, Schweiz, im Preisbereich von 5-20 Millionen CHF. Sammle Informationen zu: Ort, Preis, Beschreibung, URL zu Bildern, Verkäufer/Kontaktinformationen.", - "excelFormatPrompt": "Erstelle eine Excel-Datei mit den recherchierten Immobilien. Jede Immobilie soll eine Zeile sein mit den folgenden Spalten: Ort, Preis (in CHF), Beschreibung, URL zu Bild, Verkäufer. Verwende die Daten aus der Webrecherche." - } - }, - { - "template": { - "overview": "Spesenbelege Zusammenfassung", - "tasks": [ - { - "id": "Task03", - "title": "Spesenbelege CSV Zusammenfassung", - "description": "Liest PDF-Spesenbelege aus SharePoint-Ordner und erstellt CSV-Zusammenfassung", - "objective": "Extrahiere alle PDF-Spesenbelege aus einem SharePoint-Ordner und erstelle eine CSV-Datei mit allen Spesendaten im selben Ordner", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findDocumentPath", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "searchQuery": "{{KEY:sharepointFolderNameSource}}:files:.pdf", - "maxResults": 100 - }, - "execResultLabel": "sharepoint_pdf_files" - }, - { - "execMethod": "sharepoint", - "execAction": "readDocuments", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "pathObject": "sharepoint_pdf_files" - }, - "execResultLabel": "spesenbelege_documents" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:expenseExtractionPrompt}}", - "documentList": ["spesenbelege_documents"], - "resultType": "csv" - }, - "execResultLabel": "spesenbelege_csv" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["spesenbelege_csv"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "spesenbelege_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointFolderNameSource": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/expenses", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "expenseExtractionPrompt": "Verarbeite alle bereitgestellten Dokumente, aber extrahiere nur Daten aus PDF-Spesenbelegen (ignoriere andere Dateitypen). Für jeden gefundenen PDF-Spesenbeleg extrahiere als separaten Datensatz: Datum, Betrag, MWST %, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler/Verkäufer, Steuerbetrag. Erstelle eine CSV-Datei mit einer Zeile pro Spesenbeleg. Verwende die folgenden Spaltenüberschriften: Datum, Betrag, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler, Steuerbetrag. Stelle sicher, dass alle Beträge numerisch sind und Datumswerte im Format YYYY-MM-DD vorliegen. Wenn ein Dokument kein Spesenbeleg ist, ignoriere es." - } - }, - { - "template": { - "overview": "Preprocessing Server Data Update", - "tasks": [ - { - "id": "Task04", - "title": "Trigger Preprocessing Server", - "description": "Triggers the preprocessing server at customer tenant to update database with configuration", - "objective": "Call preprocessing server endpoint to update database with provided configuration JSON", - "actionList": [ - { - "execMethod": "context", - "execAction": "triggerPreprocessingServer", - "execParameters": { - "endpoint": "{{KEY:endpoint}}", - "configJson": "{{KEY:configJson}}", - "authSecretConfigKey": "{{KEY:authSecretConfigKey}}" - }, - "execResultLabel": "preprocessing_server_result" - } - ] - } - ] - }, - "parameters": { - "endpoint": "https://poweron-althaus-preprocess-prod-e3fegaatc7faency.switzerlandnorth-01.azurewebsites.net/api/v1/dataprocessor/update-db-with-config", - "authSecretConfigKey": "PREPROCESS_ALTHAUS_CHAT_SECRET", - "configJson": "{\"tables\":[{\"name\":\"Artikel\",\"powerbi_table_name\":\"Artikel\",\"steps\":[{\"keep\":{\"columns\":[\"I_ID\",\"Artikelbeschrieb\",\"Artikelbezeichnung\",\"Artikelgruppe\",\"Artikelkategorie\",\"Artikelkürzel\",\"Artikelnummer\",\"Einheit\",\"Gesperrt\",\"Keywords\",\"Lieferant\",\"Warengruppe\"]}},{\"fillna\":{\"column\":\"Lieferant\",\"value\":\"Unbekannt\"}}]},{\"name\":\"Einkaufspreis\",\"powerbi_table_name\":\"Einkaufspreis\",\"steps\":[{\"to_numeric\":{\"column\":\"EP_CHF\",\"errors\":\"coerce\"}},{\"dropna\":{\"subset\":[\"EP_CHF\"]}}]}]}" - } - }, - { - "template": { - "overview": "JIRA to SharePoint Ticket Synchronization", - "tasks": [ - { - "id": "Task01", - "title": "Sync JIRA Tickets to SharePoint", - "description": "Export JIRA tickets, merge with SharePoint file, upload back, and import changes to JIRA", - "objective": "Synchronize JIRA tickets with SharePoint file (bidirectional sync)", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findSiteByUrl", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "hostname": "{{KEY:sharepointHostname}}", - "sitePath": "{{KEY:sharepointSitePath}}" - }, - "execResultLabel": "sharepoint_site" - }, - { - "execMethod": "jira", - "execAction": "connectJira", - "execParameters": { - "apiUsername": "{{KEY:jiraUsername}}", - "apiTokenConfigKey": "{{KEY:jiraTokenConfigKey}}", - "apiUrl": "{{KEY:jiraUrl}}", - "projectCode": "{{KEY:jiraProjectCode}}", - "issueType": "{{KEY:jiraIssueType}}", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "jira_connection" - }, - { - "execMethod": "jira", - "execAction": "exportTicketsAsJson", - "execParameters": { - "connectionId": "jira_connection", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "jira_exported_tickets" - }, - { - "execMethod": "sharepoint", - "execAction": "downloadFileByPath", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" - }, - "execResultLabel": "existing_file_content" - }, - { - "execMethod": "jira", - "execAction": "parseExcelContent", - "execParameters": { - "excelContent": "existing_file_content", - "skipRows": 3, - "hasCustomHeaders": True - }, - "execResultLabel": "existing_parsed_data" - }, - { - "execMethod": "jira", - "execAction": "mergeTicketData", - "execParameters": { - "jiraData": "jira_exported_tickets", - "existingData": "existing_parsed_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}", - "idField": "ID" - }, - "execResultLabel": "merged_ticket_data" - }, - { - "execMethod": "sharepoint", - "execAction": "copyFile", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "sourceFolder": "{{KEY:sharepointMainFolder}}", - "sourceFile": "{{KEY:syncFileName}}", - "destFolder": "{{KEY:sharepointBackupFolder}}", - "destFile": "backup_{{TIMESTAMP}}_{{KEY:syncFileName}}" - }, - "execResultLabel": "file_backup" - }, - { - "execMethod": "jira", - "execAction": "createExcelContent", - "execParameters": { - "data": "merged_ticket_data", - "headers": "existing_parsed_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "new_file_content" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadFile", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "folderPath": "{{KEY:sharepointMainFolder}}", - "fileName": "{{KEY:syncFileName}}", - "content": "new_file_content" - }, - "execResultLabel": "uploaded_file" - }, - { - "execMethod": "sharepoint", - "execAction": "downloadFileByPath", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" - }, - "execResultLabel": "uploaded_file_content" - }, - { - "execMethod": "jira", - "execAction": "parseExcelContent", - "execParameters": { - "excelContent": "uploaded_file_content", - "skipRows": 3, - "hasCustomHeaders": True - }, - "execResultLabel": "import_data" - }, - { - "execMethod": "jira", - "execAction": "importTicketsFromJson", - "execParameters": { - "connectionId": "jira_connection", - "ticketData": "import_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "import_result" - } - ] - } - ] - }, - "parameters": { - "sharepointConnection": "connection:msft:patrick.motsch@delta.ch", - "sharepointHostname": "deltasecurityag.sharepoint.com", - "sharepointSitePath": "SteeringBPM", - "sharepointMainFolder": "/General/50 Docs hosted by SELISE", - "sharepointBackupFolder": "/General/50 Docs hosted by SELISE/SyncHistory", - "syncFileName": "DELTAgroup x SELISE Ticket Exchange List.xlsx", - "jiraUsername": "p.motsch@valueon.ch", - "jiraTokenConfigKey": "Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET", - "jiraUrl": "https://deltasecurity.atlassian.net", - "jiraProjectCode": "DCS", - "jiraIssueType": "Task", - "taskSyncDefinition": "{\"ID\":[\"get\",[\"key\"]],\"Module Category\":[\"get\",[\"fields\",\"customfield_10058\",\"value\"]],\"Summary\":[\"get\",[\"fields\",\"summary\"]],\"Description\":[\"get\",[\"fields\",\"description\"]],\"References\":[\"get\",[\"fields\",\"customfield_10066\"]],\"Priority\":[\"get\",[\"fields\",\"priority\",\"name\"]],\"Issue Status\":[\"get\",[\"fields\",\"status\",\"name\"]],\"Assignee\":[\"get\",[\"fields\",\"assignee\",\"displayName\"]],\"Issue Created\":[\"get\",[\"fields\",\"created\"]],\"Due Date\":[\"get\",[\"fields\",\"duedate\"]],\"DELTA Comments\":[\"get\",[\"fields\",\"customfield_10167\"]],\"SELISE Ticket References\":[\"put\",[\"fields\",\"customfield_10067\"]],\"SELISE Status Values\":[\"put\",[\"fields\",\"customfield_10065\"]],\"SELISE Comments\":[\"put\",[\"fields\",\"customfield_10168\"]]}" - } - }, - { - "template": { - "overview": "Expenses PDF to Trustee Position", - "tasks": [ - { - "id": "Task01", - "title": "Run trustee pipeline on SharePoint files", - "description": "Extract expenses from SharePoint PDFs, create positions + documents, sync to accounting", - "objective": "End-to-end: SharePoint folder → AI extraction → Trustee DB → Accounting sync", - "actionList": [ - { - "execMethod": "trustee", - "execAction": "extractFromFiles", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "sharepointFolder": "{{KEY:sharepointFolder}}", - "featureInstanceId": "{{KEY:featureInstanceId}}" - }, - "execResultLabel": "extract_result" - }, - { - "execMethod": "trustee", - "execAction": "processDocuments", - "execParameters": { - "documentList": "docList:{{PREV_MESSAGE_ID}}:extract_result", - "featureInstanceId": "{{KEY:featureInstanceId}}" - }, - "execResultLabel": "process_result" - }, - { - "execMethod": "trustee", - "execAction": "syncToAccounting", - "execParameters": { - "documentList": "docList:{{PREV_MESSAGE_ID}}:process_result", - "featureInstanceId": "{{KEY:featureInstanceId}}" - }, - "execResultLabel": "sync_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "", - "sharepointFolder": "", - "featureInstanceId": "" - } - } - ] -} - - -def getAutomationTemplates() -> Dict[str, Any]: - """ - Get automation templates. - - Returns: - Dict containing the automation templates structure with 'sets' key. - """ - return AUTOMATION_TEMPLATES - diff --git a/modules/features/automation/subAutomationUtils.py b/modules/features/automation/subAutomationUtils.py deleted file mode 100644 index 97d28719..00000000 --- a/modules/features/automation/subAutomationUtils.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Utility functions for automation feature. - -Moved from interfaces/interfaceDbChat.py. -""" - -import json -from typing import Dict, Any -from datetime import datetime, UTC - - -def parseScheduleToCron(schedule: str) -> Dict[str, Any]: - """Parse schedule string to cron kwargs for APScheduler""" - parts = schedule.split() - if len(parts) != 5: - raise ValueError(f"Invalid schedule format: {schedule}") - - return { - "minute": parts[0], - "hour": parts[1], - "day": parts[2], - "month": parts[3], - "day_of_week": parts[4] - } - - -def planToPrompt(plan: Dict) -> str: - """Convert plan structure to prompt string for workflow execution""" - return plan.get("userMessage", plan.get("overview", "Execute automation workflow")) - - -def replacePlaceholders(template: str, placeholders: Dict[str, str]) -> str: - """Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}} or {{TIMESTAMP}}""" - result = template - - # Replace TIMESTAMP placeholder first (calculated placeholder, not from parameters) - timestampPattern = "{{TIMESTAMP}}" - if timestampPattern in result: - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") - result = result.replace(timestampPattern, timestamp) - - for placeholderName, value in placeholders.items(): - pattern = f"{{{{KEY:{placeholderName}}}}}" - - # Check if placeholder is in an array context like ["{{KEY:...}}"] - # If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array - arrayPattern = f'["{pattern}"]' - if arrayPattern in result: - # Check if value is a JSON array/dict - isArrayValue = False - arrayValue = None - - if isinstance(value, (list, dict)): - isArrayValue = True - arrayValue = json.dumps(value) - elif isinstance(value, str): - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - isArrayValue = True - arrayValue = value # Already valid JSON string - except (json.JSONDecodeError, ValueError): - pass - - if isArrayValue: - # Replace ["{{KEY:...}}"] with the array value - result = result.replace(arrayPattern, arrayValue) - continue # Skip the regular replacement below - - # Regular replacement - check if in quoted context - patternStart = result.find(pattern) - isQuoted = False - if patternStart > 0: - charBefore = result[patternStart - 1] if patternStart > 0 else None - patternEnd = patternStart + len(pattern) - charAfter = result[patternEnd] if patternEnd < len(result) else None - if charBefore == '"' and charAfter == '"': - isQuoted = True - - # Handle different value types - if isinstance(value, (list, dict)): - # Python list/dict - convert to JSON - replacement = json.dumps(value) - elif isinstance(value, str): - # String value - check if it's a JSON string representing list/dict - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - # It's a JSON string of a list/dict - if isQuoted: - # In quoted context, escape the JSON string - escaped = json.dumps(value) - replacement = escaped[1:-1] # Remove outer quotes - else: - # In unquoted context, use JSON directly - replacement = value - else: - # It's a JSON string of a primitive - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - except (json.JSONDecodeError, ValueError): - # Not valid JSON - treat as plain string - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - else: - # Numbers, booleans, None - convert to string - replacement = str(value) - result = result.replace(pattern, replacement) - return result - diff --git a/modules/features/automation2/__init__.py b/modules/features/automation2/__init__.py deleted file mode 100644 index c86d7e61..00000000 --- a/modules/features/automation2/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# Automation2 feature - n8n-style flow automation (backup/parallel to legacy automation) diff --git a/modules/features/automation2/datamodelFeatureAutomation2.py b/modules/features/automation2/datamodelFeatureAutomation2.py deleted file mode 100644 index 97b33754..00000000 --- a/modules/features/automation2/datamodelFeatureAutomation2.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Automation2 models: Automation2Workflow, Automation2WorkflowRun, Automation2HumanTask.""" - -from typing import Dict, Any, List, Optional -from pydantic import BaseModel, Field -from modules.datamodels.datamodelBase import PowerOnModel -from modules.shared.attributeUtils import registerModelLabels -import uuid - - -class Automation2Workflow(BaseModel): - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - mandateId: str = Field( - description="Mandate ID", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - featureInstanceId: str = Field( - description="Feature instance ID", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - label: str = Field( - description="User-friendly workflow name", - json_schema_extra={"frontend_type": "text", "frontend_required": True}, - ) - graph: Dict[str, Any] = Field( - default_factory=dict, - description="Graph with nodes and connections (incl. node parameters)", - json_schema_extra={"frontend_type": "textarea", "frontend_required": True}, - ) - active: bool = Field( - default=True, - description="Whether workflow is active", - json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}, - ) - invocations: List[Dict[str, Any]] = Field( - default_factory=list, - description="Entry points / starts (manual, form, schedule, webhook, …) configured outside the canvas", - json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, - ) - - -registerModelLabels( - "Automation2Workflow", - {"en": "Automation2 Workflow", "de": "Automation2 Workflow", "fr": "Workflow Automation2"}, - { - "id": {"en": "ID", "de": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "de": "Feature-Instanz-ID", "fr": "ID instance"}, - "label": {"en": "Label", "de": "Bezeichnung", "fr": "Libellé"}, - "graph": {"en": "Graph", "de": "Graph", "fr": "Graphe"}, - "active": {"en": "Active", "de": "Aktiv", "fr": "Actif"}, - "invocations": {"en": "Starts / Entry points", "de": "Starts / Einstiegspunkte", "fr": "Points d'entrée"}, - }, -) - - -class Automation2WorkflowRun(PowerOnModel): - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - workflowId: str = Field( - description="Workflow ID", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, - ) - status: str = Field( - default="running", - description="Status: running|paused|completed|failed", - json_schema_extra={"frontend_type": "text", "frontend_required": False}, - ) - nodeOutputs: Dict[str, Any] = Field( - default_factory=dict, - description="Outputs from executed nodes", - json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, - ) - currentNodeId: Optional[str] = Field( - default=None, - description="Node ID when paused (human task)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - context: Dict[str, Any] = Field( - default_factory=dict, - description="Context for resume (connectionMap, inputSources, etc.)", - json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, - ) - - -registerModelLabels( - "Automation2WorkflowRun", - {"en": "Automation2 Workflow Run", "de": "Automation2 Workflow-Ausführung", "fr": "Exécution workflow"}, - { - "id": {"en": "ID", "de": "ID", "fr": "ID"}, - "workflowId": {"en": "Workflow ID", "de": "Workflow-ID", "fr": "ID workflow"}, - "status": {"en": "Status", "de": "Status", "fr": "Statut"}, - "nodeOutputs": {"en": "Node Outputs", "de": "Node-Ausgaben", "fr": "Sorties nœuds"}, - "currentNodeId": {"en": "Current Node", "de": "Aktueller Knoten", "fr": "Nœud actuel"}, - "context": {"en": "Context", "de": "Kontext", "fr": "Contexte"}, - }, -) - - -class Automation2HumanTask(PowerOnModel): - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, - ) - runId: str = Field( - description="Workflow run ID", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, - ) - workflowId: str = Field( - description="Workflow ID", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, - ) - nodeId: str = Field( - description="Node ID in the graph", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, - ) - nodeType: str = Field( - description="Node type: form|approval|upload|comment|review|selection|confirmation", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, - ) - config: Dict[str, Any] = Field( - default_factory=dict, - description="Node config (form schema, approval text, etc.)", - json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, - ) - assigneeId: Optional[str] = Field( - default=None, - description="User ID assigned to complete the task", - json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}, - ) - status: str = Field( - default="pending", - description="Status: pending|completed|rejected", - json_schema_extra={"frontend_type": "text", "frontend_required": False}, - ) - result: Optional[Dict[str, Any]] = Field( - default=None, - description="Task result (form data, approval decision, etc.)", - json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, - ) - - -registerModelLabels( - "Automation2HumanTask", - {"en": "Automation2 Human Task", "de": "Automation2 Benutzer-Aufgabe", "fr": "Tâche utilisateur"}, - { - "id": {"en": "ID", "de": "ID", "fr": "ID"}, - "runId": {"en": "Run ID", "de": "Lauf-ID", "fr": "ID exécution"}, - "workflowId": {"en": "Workflow ID", "de": "Workflow-ID", "fr": "ID workflow"}, - "nodeId": {"en": "Node ID", "de": "Knoten-ID", "fr": "ID nœud"}, - "nodeType": {"en": "Node Type", "de": "Knotentyp", "fr": "Type nœud"}, - "config": {"en": "Config", "de": "Konfiguration", "fr": "Configuration"}, - "assigneeId": {"en": "Assignee", "de": "Zugewiesen an", "fr": "Assigné à"}, - "status": {"en": "Status", "de": "Status", "fr": "Statut"}, - "result": {"en": "Result", "de": "Ergebnis", "fr": "Résultat"}, - }, -) diff --git a/modules/features/graphicalEditor/__init__.py b/modules/features/graphicalEditor/__init__.py new file mode 100644 index 00000000..bb8c0a4b --- /dev/null +++ b/modules/features/graphicalEditor/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2025 Patrick Motsch +# GraphicalEditor feature - n8n-style flow automation with visual editor diff --git a/modules/features/graphicalEditor/datamodelFeatureGraphicalEditor.py b/modules/features/graphicalEditor/datamodelFeatureGraphicalEditor.py new file mode 100644 index 00000000..bf94c12a --- /dev/null +++ b/modules/features/graphicalEditor/datamodelFeatureGraphicalEditor.py @@ -0,0 +1,482 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""GraphicalEditor models with Auto-prefix: AutoWorkflow, AutoVersion, AutoRun, AutoStepLog, AutoTask.""" + +from enum import Enum +from typing import Dict, Any, List, Optional +from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel +from modules.shared.attributeUtils import registerModelLabels +import uuid + + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + +class AutoWorkflowStatus(str, Enum): + DRAFT = "draft" + PUBLISHED = "published" + ARCHIVED = "archived" + + +class AutoRunStatus(str, Enum): + RUNNING = "running" + PAUSED = "paused" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class AutoStepStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + SKIPPED = "skipped" + + +class AutoTaskStatus(str, Enum): + PENDING = "pending" + COMPLETED = "completed" + CANCELLED = "cancelled" + EXPIRED = "expired" + + +class AutoTemplateScope(str, Enum): + USER = "user" + INSTANCE = "instance" + MANDATE = "mandate" + SYSTEM = "system" + + +# --------------------------------------------------------------------------- +# AutoWorkflow +# --------------------------------------------------------------------------- + +class AutoWorkflow(PowerOnModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + mandateId: str = Field( + description="Mandate ID", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + featureInstanceId: str = Field( + description="Feature instance ID", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + label: str = Field( + description="User-friendly workflow name", + json_schema_extra={"frontend_type": "text", "frontend_required": True}, + ) + description: Optional[str] = Field( + default=None, + description="Workflow description", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + tags: List[str] = Field( + default_factory=list, + description="Tags for categorization", + json_schema_extra={"frontend_type": "tags", "frontend_required": False}, + ) + isTemplate: bool = Field( + default=False, + description="Whether this workflow is a template", + json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}, + ) + templateSourceId: Optional[str] = Field( + default=None, + description="ID of the template this workflow was created from", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + templateScope: Optional[str] = Field( + default=None, + description="Template scope: user, instance, mandate, system (AutoTemplateScope)", + json_schema_extra={"frontend_type": "select", "frontend_required": False}, + ) + sharedReadOnly: bool = Field( + default=False, + description="If true, shared template is read-only for non-owners", + json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}, + ) + currentVersionId: Optional[str] = Field( + default=None, + description="ID of the currently published AutoVersion", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + active: bool = Field( + default=True, + description="Whether workflow is active", + json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}, + ) + eventId: Optional[str] = Field( + default=None, + description="Scheduler event ID for incremental sync", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + notifyOnFailure: bool = Field( + default=True, + description="Send notification (in-app + email) when a run fails", + json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}, + ) + # Legacy fields kept for backward compatibility during transition + graph: Dict[str, Any] = Field( + default_factory=dict, + description="Graph with nodes and connections (legacy; prefer AutoVersion.graph)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + invocations: List[Dict[str, Any]] = Field( + default_factory=list, + description="Entry points / starts (manual, form, schedule, webhook, ...)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + + +registerModelLabels( + "AutoWorkflow", + {"en": "Workflow", "de": "Workflow", "fr": "Workflow"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "de": "Feature-Instanz-ID", "fr": "ID instance"}, + "label": {"en": "Label", "de": "Bezeichnung", "fr": "Libellé"}, + "description": {"en": "Description", "de": "Beschreibung", "fr": "Description"}, + "tags": {"en": "Tags", "de": "Tags", "fr": "Tags"}, + "isTemplate": {"en": "Is Template", "de": "Ist Vorlage", "fr": "Est modèle"}, + "templateSourceId": {"en": "Template Source", "de": "Vorlagen-Quelle", "fr": "Source du modèle"}, + "templateScope": {"en": "Template Scope", "de": "Vorlagen-Bereich", "fr": "Portée du modèle"}, + "sharedReadOnly": {"en": "Shared Read-Only", "de": "Freigabe nur-lesen", "fr": "Partage lecture seule"}, + "currentVersionId": {"en": "Current Version", "de": "Aktuelle Version", "fr": "Version actuelle"}, + "active": {"en": "Active", "de": "Aktiv", "fr": "Actif"}, + "eventId": {"en": "Event ID", "de": "Event-ID", "fr": "ID événement"}, + "graph": {"en": "Graph", "de": "Graph", "fr": "Graphe"}, + "invocations": {"en": "Starts / Entry points", "de": "Starts / Einstiegspunkte", "fr": "Points d'entrée"}, + }, +) + + +# --------------------------------------------------------------------------- +# AutoVersion +# --------------------------------------------------------------------------- + +class AutoVersion(PowerOnModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + workflowId: str = Field( + description="FK -> AutoWorkflow", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + versionNumber: int = Field( + default=1, + description="Incrementing version number", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + status: str = Field( + default=AutoWorkflowStatus.DRAFT.value, + description="Version status: draft, published, archived", + json_schema_extra={"frontend_type": "select", "frontend_required": False}, + ) + graph: Dict[str, Any] = Field( + default_factory=dict, + description="Graph with nodes and connections (incl. node parameters)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": True}, + ) + invocations: List[Dict[str, Any]] = Field( + default_factory=list, + description="Entry points / starts for this version", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + publishedAt: Optional[float] = Field( + default=None, + description="Timestamp when version was published", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False}, + ) + publishedBy: Optional[str] = Field( + default=None, + description="User ID who published this version", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + + +registerModelLabels( + "AutoVersion", + {"en": "Workflow Version", "de": "Workflow-Version", "fr": "Version workflow"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "de": "Workflow-ID", "fr": "ID workflow"}, + "versionNumber": {"en": "Version", "de": "Version", "fr": "Version"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "graph": {"en": "Graph", "de": "Graph", "fr": "Graphe"}, + "invocations": {"en": "Entry Points", "de": "Einstiegspunkte", "fr": "Points d'entrée"}, + "publishedAt": {"en": "Published At", "de": "Veröffentlicht am", "fr": "Publié le"}, + "publishedBy": {"en": "Published By", "de": "Veröffentlicht von", "fr": "Publié par"}, + }, +) + + +# --------------------------------------------------------------------------- +# AutoRun +# --------------------------------------------------------------------------- + +class AutoRun(PowerOnModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + workflowId: str = Field( + description="Workflow ID", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + versionId: Optional[str] = Field( + default=None, + description="AutoVersion ID used for this run", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + status: str = Field( + default=AutoRunStatus.RUNNING.value, + description="Status: running, paused, completed, failed, cancelled", + json_schema_extra={"frontend_type": "text", "frontend_required": False}, + ) + trigger: Dict[str, Any] = Field( + default_factory=dict, + description="Trigger info (type, entryPointId, payload, etc.)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + startedAt: Optional[float] = Field( + default=None, + description="Run start timestamp", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False}, + ) + completedAt: Optional[float] = Field( + default=None, + description="Run completion timestamp", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False}, + ) + nodeOutputs: Dict[str, Any] = Field( + default_factory=dict, + description="Outputs from executed nodes", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + currentNodeId: Optional[str] = Field( + default=None, + description="Node ID when paused (human task / email wait)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + resumeContext: Dict[str, Any] = Field( + default_factory=dict, + description="Context for resume (connectionMap, inputSources, etc.)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + error: Optional[str] = Field( + default=None, + description="Error message if failed", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False}, + ) + costTokens: int = Field( + default=0, + description="Total tokens consumed by AI nodes", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + costCredits: float = Field( + default=0.0, + description="Total credits consumed", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + + +registerModelLabels( + "AutoRun", + {"en": "Workflow Run", "de": "Workflow-Ausführung", "fr": "Exécution workflow"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "de": "Workflow-ID", "fr": "ID workflow"}, + "versionId": {"en": "Version ID", "de": "Versions-ID", "fr": "ID version"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "trigger": {"en": "Trigger", "de": "Auslöser", "fr": "Déclencheur"}, + "startedAt": {"en": "Started At", "de": "Gestartet am", "fr": "Démarré le"}, + "completedAt": {"en": "Completed At", "de": "Abgeschlossen am", "fr": "Terminé le"}, + "nodeOutputs": {"en": "Node Outputs", "de": "Node-Ausgaben", "fr": "Sorties nœuds"}, + "currentNodeId": {"en": "Current Node", "de": "Aktueller Knoten", "fr": "Nœud actuel"}, + "resumeContext": {"en": "Resume Context", "de": "Wiederaufnahme-Kontext", "fr": "Contexte reprise"}, + "error": {"en": "Error", "de": "Fehler", "fr": "Erreur"}, + "costTokens": {"en": "Tokens Used", "de": "Verbrauchte Tokens", "fr": "Tokens utilisés"}, + "costCredits": {"en": "Credits Used", "de": "Verbrauchte Credits", "fr": "Crédits utilisés"}, + }, +) + + +# --------------------------------------------------------------------------- +# AutoStepLog +# --------------------------------------------------------------------------- + +class AutoStepLog(PowerOnModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + runId: str = Field( + description="FK -> AutoRun", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + nodeId: str = Field( + description="Node ID in the graph", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + nodeType: str = Field( + description="Node type (e.g. ai.chat, email.send)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + status: str = Field( + default=AutoStepStatus.PENDING.value, + description="Step status: pending, running, completed, failed, skipped", + json_schema_extra={"frontend_type": "text", "frontend_required": False}, + ) + inputSnapshot: Dict[str, Any] = Field( + default_factory=dict, + description="Snapshot of inputs at execution time", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + output: Dict[str, Any] = Field( + default_factory=dict, + description="Node output", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + error: Optional[str] = Field( + default=None, + description="Error message if step failed", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False}, + ) + startedAt: Optional[float] = Field( + default=None, + description="Step start timestamp", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False}, + ) + completedAt: Optional[float] = Field( + default=None, + description="Step completion timestamp", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False}, + ) + durationMs: Optional[int] = Field( + default=None, + description="Execution duration in milliseconds", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + tokensUsed: int = Field( + default=0, + description="Tokens consumed by this step", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + retryCount: int = Field( + default=0, + description="Number of retries executed", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False}, + ) + + +registerModelLabels( + "AutoStepLog", + {"en": "Step Log", "de": "Schritt-Protokoll", "fr": "Journal d'étape"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "runId": {"en": "Run ID", "de": "Lauf-ID", "fr": "ID exécution"}, + "nodeId": {"en": "Node ID", "de": "Knoten-ID", "fr": "ID nœud"}, + "nodeType": {"en": "Node Type", "de": "Knotentyp", "fr": "Type nœud"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "inputSnapshot": {"en": "Input Snapshot", "de": "Eingabe-Snapshot", "fr": "Snapshot entrée"}, + "output": {"en": "Output", "de": "Ausgabe", "fr": "Sortie"}, + "error": {"en": "Error", "de": "Fehler", "fr": "Erreur"}, + "startedAt": {"en": "Started At", "de": "Gestartet am", "fr": "Démarré le"}, + "completedAt": {"en": "Completed At", "de": "Abgeschlossen am", "fr": "Terminé le"}, + "durationMs": {"en": "Duration (ms)", "de": "Dauer (ms)", "fr": "Durée (ms)"}, + "tokensUsed": {"en": "Tokens Used", "de": "Verbrauchte Tokens", "fr": "Tokens utilisés"}, + "retryCount": {"en": "Retry Count", "de": "Wiederholungen", "fr": "Nombre de tentatives"}, + }, +) + + +# --------------------------------------------------------------------------- +# AutoTask +# --------------------------------------------------------------------------- + +class AutoTask(PowerOnModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}, + ) + runId: str = Field( + description="FK -> AutoRun", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + workflowId: str = Field( + description="Workflow ID", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + nodeId: str = Field( + description="Node ID in the graph", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + nodeType: str = Field( + description="Node type: form, approval, upload, comment, review, selection, confirmation", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}, + ) + config: Dict[str, Any] = Field( + default_factory=dict, + description="Node config (form schema, approval text, etc.)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + assigneeId: Optional[str] = Field( + default=None, + description="User ID assigned to complete the task", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}, + ) + status: str = Field( + default=AutoTaskStatus.PENDING.value, + description="Status: pending, completed, cancelled, expired", + json_schema_extra={"frontend_type": "text", "frontend_required": False}, + ) + result: Optional[Dict[str, Any]] = Field( + default=None, + description="Task result (form data, approval decision, etc.)", + json_schema_extra={"frontend_type": "textarea", "frontend_required": False}, + ) + expiresAt: Optional[float] = Field( + default=None, + description="Expiration timestamp for the task", + json_schema_extra={"frontend_type": "datetime", "frontend_required": False}, + ) + + +registerModelLabels( + "AutoTask", + {"en": "Task", "de": "Aufgabe", "fr": "Tâche"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "runId": {"en": "Run ID", "de": "Lauf-ID", "fr": "ID exécution"}, + "workflowId": {"en": "Workflow ID", "de": "Workflow-ID", "fr": "ID workflow"}, + "nodeId": {"en": "Node ID", "de": "Knoten-ID", "fr": "ID nœud"}, + "nodeType": {"en": "Node Type", "de": "Knotentyp", "fr": "Type nœud"}, + "config": {"en": "Config", "de": "Konfiguration", "fr": "Configuration"}, + "assigneeId": {"en": "Assignee", "de": "Zugewiesen an", "fr": "Assigné à"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "result": {"en": "Result", "de": "Ergebnis", "fr": "Résultat"}, + "expiresAt": {"en": "Expires At", "de": "Läuft ab am", "fr": "Expire le"}, + }, +) + + +# --------------------------------------------------------------------------- +# Backward-compatible aliases for transition period +# --------------------------------------------------------------------------- + +Automation2Workflow = AutoWorkflow +Automation2WorkflowRun = AutoRun +Automation2HumanTask = AutoTask diff --git a/modules/features/automation2/emailPoller.py b/modules/features/graphicalEditor/emailPoller.py similarity index 97% rename from modules/features/automation2/emailPoller.py rename to modules/features/graphicalEditor/emailPoller.py index ca440ca2..7c769463 100644 --- a/modules/features/automation2/emailPoller.py +++ b/modules/features/graphicalEditor/emailPoller.py @@ -25,8 +25,8 @@ async def _pollEmailWaits(eventUser) -> None: Stops the poller when no runs are waiting. """ try: - from modules.features.automation2.interfaceFeatureAutomation2 import getAutomation2Interface - from modules.features.automation2.mainAutomation2 import getAutomation2Services + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface as getAutomation2Interface + from modules.features.graphicalEditor.mainGraphicalEditor import getGraphicalEditorServices as getAutomation2Services from modules.workflows.automation2.executionEngine import executeGraph from modules.workflows.processing.shared.methodDiscovery import discoverMethods from modules.interfaces.interfaceDbApp import getRootInterface diff --git a/modules/features/automation2/entryPoints.py b/modules/features/graphicalEditor/entryPoints.py similarity index 100% rename from modules/features/automation2/entryPoints.py rename to modules/features/graphicalEditor/entryPoints.py diff --git a/modules/features/automation2/interfaceFeatureAutomation2.py b/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py similarity index 62% rename from modules/features/automation2/interfaceFeatureAutomation2.py rename to modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py index cec51181..c777ba59 100644 --- a/modules/features/automation2/interfaceFeatureAutomation2.py +++ b/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py @@ -1,8 +1,8 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Interface for Automation2 feature - Workflows, Runs, Human Tasks. -Uses PostgreSQL poweron_automation2 database. +Interface for GraphicalEditor feature - Workflows, Runs, Human Tasks. +Uses PostgreSQL poweron_graphicaleditor database (Greenfield). """ import base64 @@ -25,38 +25,50 @@ def _make_json_serializable(obj: Any) -> Any: return obj from modules.datamodels.datamodelUam import User -from modules.features.automation2.datamodelFeatureAutomation2 import ( - Automation2Workflow, - Automation2WorkflowRun, - Automation2HumanTask, +from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import ( + AutoWorkflow, + AutoVersion, + AutoRun, + AutoStepLog, + AutoTask, + AutoWorkflow as Automation2Workflow, + AutoRun as Automation2WorkflowRun, + AutoTask as Automation2HumanTask, ) -from modules.features.automation2.entryPoints import normalize_invocations_list +from modules.features.graphicalEditor.entryPoints import normalize_invocations_list from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) +_GREENFIELD_DB = "poweron_graphicaleditor" +_CALLBACK_WORKFLOW_CHANGED = "graphicalEditor.workflow.changed" -def getAutomation2Interface( + +def getGraphicalEditorInterface( currentUser: User, mandateId: str, featureInstanceId: str, -) -> "Automation2Objects": - """Factory for Automation2 interface with user context.""" - return Automation2Objects( +) -> "GraphicalEditorObjects": + """Factory for GraphicalEditor interface with user context.""" + return GraphicalEditorObjects( currentUser=currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId, ) +# Backward-compatible alias used by workflows/automation2/ execution engine +getAutomation2Interface = getGraphicalEditorInterface + + def getAllWorkflowsForScheduling() -> List[Dict[str, Any]]: """ - Get all active Automation2 workflows that have a schedule entry point (primary invocation). + Get all active workflows that have a schedule entry point (primary invocation). Used by the scheduler to register cron jobs. Does not filter by mandate/instance. """ dbHost = APP_CONFIG.get("DB_HOST", "localhost") - dbDatabase = "poweron_automation2" + dbDatabase = _GREENFIELD_DB dbUser = APP_CONFIG.get("DB_USER") dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD") dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) @@ -69,10 +81,8 @@ def getAllWorkflowsForScheduling() -> List[Dict[str, Any]]: userId=None, ) if not connector._ensureTableExists(Automation2Workflow): - logger.warning("Automation2 schedule: table Automation2Workflow does not exist") + logger.warning("GraphicalEditor schedule: table Automation2Workflow does not exist yet") return [] - # Don't filter by active in SQL: existing workflows may have active=NULL. - # Treat NULL as active; skip only when active is explicitly False. records = connector.getRecordset( Automation2Workflow, recordFilter=None, @@ -89,7 +99,6 @@ def getAllWorkflowsForScheduling() -> List[Dict[str, Any]]: if not isinstance(primary, dict): primary = {} - # Cron comes from graph start node params (trigger.schedule) graph = wf.get("graph") or {} nodes = graph.get("nodes") or [] cron = None @@ -103,7 +112,6 @@ def getAllWorkflowsForScheduling() -> List[Dict[str, Any]]: if not cron or not isinstance(cron, str) or not cron.strip(): continue - # Prefer invocations; if graph has trigger.schedule but invocations say manual, still schedule if primary.get("kind") == "schedule" and primary.get("enabled", True): entry_point_id = primary.get("id") elif invocations and isinstance(invocations[0], dict) and invocations[0].get("id"): @@ -120,15 +128,15 @@ def getAllWorkflowsForScheduling() -> List[Dict[str, Any]]: "workflow": wf, }) logger.info( - "Automation2 schedule: DB has %d workflow(s), %d active with trigger.schedule+cron", + "GraphicalEditor schedule: DB has %d workflow(s), %d active with trigger.schedule+cron", raw_count, len(result), ) return result -class Automation2Objects: - """Interface for Automation2 database operations.""" +class GraphicalEditorObjects: + """Interface for GraphicalEditor database operations (Greenfield DB).""" def __init__( self, @@ -145,9 +153,9 @@ class Automation2Objects: self.db.updateContext(self.userId) def _init_db(self): - """Initialize database connection to poweron_automation2.""" + """Initialize database connection to poweron_graphicaleditor (Greenfield).""" dbHost = APP_CONFIG.get("DB_HOST", "localhost") - dbDatabase = "poweron_automation2" + dbDatabase = _GREENFIELD_DB dbUser = APP_CONFIG.get("DB_USER") dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD") dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) @@ -159,16 +167,14 @@ class Automation2Objects: dbPort=dbPort, userId=self.userId, ) - logger.debug("Automation2 database initialized for user %s", self.userId) + logger.debug("GraphicalEditor database initialized for user %s", self.userId) # ------------------------------------------------------------------------- # Workflow CRUD # ------------------------------------------------------------------------- def getWorkflows(self, active: Optional[bool] = None) -> List[Dict[str, Any]]: - """Get all workflows for this mandate and feature instance. - Optional active filter: True=only active, False=only inactive, None=all. - """ + """Get all workflows for this mandate and feature instance.""" if not self.db._ensureTableExists(Automation2Workflow): return [] rf: Dict[str, Any] = { @@ -218,7 +224,7 @@ class Automation2Objects: out["invocations"] = normalize_invocations_list(out.get("invocations")) try: from modules.shared.callbackRegistry import callbackRegistry - callbackRegistry.trigger("automation2.workflow.changed") + callbackRegistry.trigger(_CALLBACK_WORKFLOW_CHANGED) except Exception: pass return out @@ -228,7 +234,6 @@ class Automation2Objects: existing = self.getWorkflow(workflowId) if not existing: return None - # Don't overwrite mandateId/featureInstanceId data.pop("mandateId", None) data.pop("featureInstanceId", None) if "invocations" in data: @@ -238,7 +243,7 @@ class Automation2Objects: out["invocations"] = normalize_invocations_list(out.get("invocations")) try: from modules.shared.callbackRegistry import callbackRegistry - callbackRegistry.trigger("automation2.workflow.changed") + callbackRegistry.trigger(_CALLBACK_WORKFLOW_CHANGED) except Exception: pass return out @@ -251,7 +256,7 @@ class Automation2Objects: self.db.recordDelete(Automation2Workflow, workflowId) try: from modules.shared.callbackRegistry import callbackRegistry - callbackRegistry.trigger("automation2.workflow.changed") + callbackRegistry.trigger(_CALLBACK_WORKFLOW_CHANGED) except Exception: pass return True @@ -322,7 +327,7 @@ class Automation2Objects: return [dict(r) for r in records] if records else [] def getRecentCompletedRuns(self, limit: int = 20) -> List[Dict[str, Any]]: - """Get recently completed runs for workflows in this instance (for output display).""" + """Get recently completed runs for workflows in this instance.""" if not self.db._ensureTableExists(Automation2WorkflowRun): return [] workflows = self.getWorkflows() @@ -426,10 +431,7 @@ class Automation2Objects: status: str = None, assigneeId: str = None, ) -> List[Dict[str, Any]]: - """Get tasks with optional filters. - When assigneeId is set: returns tasks assigned to that user OR unassigned (so schedule tasks show up). - When assigneeId is None: returns all tasks. - """ + """Get tasks with optional filters.""" if not self.db._ensureTableExists(Automation2HumanTask): return [] base_rf: Dict[str, Any] = {} @@ -461,3 +463,171 @@ class Automation2Objects: workflows = {w["id"]: w for w in self.getWorkflows()} filtered = [t for t in items if t.get("workflowId") in workflows] return filtered + + + # ------------------------------------------------------------------------- + # Versions (AutoVersion Lifecycle) + # ------------------------------------------------------------------------- + + def getVersions(self, workflowId: str) -> List[Dict[str, Any]]: + """Get all versions for a workflow, ordered by versionNumber desc.""" + if not self.db._ensureTableExists(AutoVersion): + return [] + records = self.db.getRecordset(AutoVersion, recordFilter={"workflowId": workflowId}) + versions = [dict(r) for r in records] if records else [] + versions.sort(key=lambda v: v.get("versionNumber", 0), reverse=True) + return versions + + def getVersion(self, versionId: str) -> Optional[Dict[str, Any]]: + """Get a single version by ID.""" + if not self.db._ensureTableExists(AutoVersion): + return None + record = self.db.getRecord(AutoVersion, versionId) + return dict(record) if record else None + + def createDraftVersion(self, workflowId: str) -> Optional[Dict[str, Any]]: + """Create a new draft version from the workflow's current graph.""" + wf = self.getWorkflow(workflowId) + if not wf: + return None + existing = self.getVersions(workflowId) + nextNumber = max((v.get("versionNumber", 0) for v in existing), default=0) + 1 + import time + data = { + "id": str(uuid.uuid4()), + "workflowId": workflowId, + "versionNumber": nextNumber, + "status": "draft", + "graph": wf.get("graph", {}), + "invocations": wf.get("invocations", []), + } + created = self.db.recordCreate(AutoVersion, data) + return dict(created) + + def publishVersion(self, versionId: str, userId: str = None) -> Optional[Dict[str, Any]]: + """Publish a draft version. Archives the previously published version.""" + version = self.getVersion(versionId) + if not version or version.get("status") != "draft": + return None + workflowId = version.get("workflowId") + existing = self.getVersions(workflowId) + for v in existing: + if v.get("status") == "published" and v.get("id") != versionId: + self.db.recordModify(AutoVersion, v["id"], {"status": "archived"}) + import time + updated = self.db.recordModify(AutoVersion, versionId, { + "status": "published", + "publishedAt": time.time(), + "publishedBy": userId, + }) + if workflowId: + self.db.recordModify(AutoWorkflow, workflowId, { + "currentVersionId": versionId, + "graph": version.get("graph", {}), + "invocations": version.get("invocations", []), + }) + return dict(updated) + + def unpublishVersion(self, versionId: str) -> Optional[Dict[str, Any]]: + """Revert a published version back to draft status.""" + version = self.getVersion(versionId) + if not version or version.get("status") != "published": + return None + workflowId = version.get("workflowId") + updated = self.db.recordModify(AutoVersion, versionId, { + "status": "draft", + "publishedAt": None, + "publishedBy": None, + }) + if workflowId: + self.db.recordModify(AutoWorkflow, workflowId, {"currentVersionId": None}) + return dict(updated) + + def archiveVersion(self, versionId: str) -> Optional[Dict[str, Any]]: + """Archive a version.""" + version = self.getVersion(versionId) + if not version: + return None + updated = self.db.recordModify(AutoVersion, versionId, {"status": "archived"}) + return dict(updated) + + # ------------------------------------------------------------------------- + # Templates + # ------------------------------------------------------------------------- + + def getTemplates(self, scope: str = None) -> List[Dict[str, Any]]: + """Get workflow templates, optionally filtered by scope.""" + if not self.db._ensureTableExists(AutoWorkflow): + return [] + rf: Dict[str, Any] = { + "mandateId": self.mandateId, + "featureInstanceId": self.featureInstanceId, + "isTemplate": True, + } + if scope: + rf["templateScope"] = scope + records = self.db.getRecordset(AutoWorkflow, recordFilter=rf) + return [dict(r) for r in records] if records else [] + + def createTemplateFromWorkflow(self, workflowId: str, scope: str = "user") -> Optional[Dict[str, Any]]: + """Create a template by copying the published AutoVersion's graph (or workflow graph as fallback).""" + wf = self.getWorkflow(workflowId) + if not wf: + return None + graph = wf.get("graph", {}) + invocations = wf.get("invocations", []) + currentVersionId = wf.get("currentVersionId") + if currentVersionId: + version = self.getVersion(currentVersionId) + if version: + graph = version.get("graph", graph) + invocations = version.get("invocations", invocations) + data = { + "id": str(uuid.uuid4()), + "mandateId": self.mandateId, + "featureInstanceId": self.featureInstanceId, + "label": f"{wf.get('label', 'Workflow')} (Template)", + "graph": graph, + "invocations": invocations, + "isTemplate": True, + "templateScope": scope, + "templateSourceId": workflowId, + "active": False, + } + created = self.db.recordCreate(AutoWorkflow, data) + return dict(created) + + def copyTemplateToUser(self, templateId: str) -> Optional[Dict[str, Any]]: + """Copy a template to a new user-owned workflow with templateScope='user'.""" + template = self.getWorkflow(templateId) + if not template or not template.get("isTemplate"): + return None + data = { + "id": str(uuid.uuid4()), + "mandateId": self.mandateId, + "featureInstanceId": self.featureInstanceId, + "label": template.get("label", "Workflow").replace(" (Template)", ""), + "graph": template.get("graph", {}), + "invocations": template.get("invocations", []), + "isTemplate": False, + "templateSourceId": templateId, + "templateScope": "user", + "active": True, + } + created = self.db.recordCreate(AutoWorkflow, data) + return dict(created) + + def shareTemplate(self, templateId: str, scope: str) -> Optional[Dict[str, Any]]: + """Share a template by changing its scope and setting sharedReadOnly.""" + template = self.getWorkflow(templateId) + if not template or not template.get("isTemplate"): + return None + updated = self.db.recordModify(AutoWorkflow, templateId, { + "templateScope": scope, + "sharedReadOnly": True, + }) + return dict(updated) + + +# Backward-compatible alias +Automation2Objects = GraphicalEditorObjects diff --git a/modules/features/automation2/mainAutomation2.py b/modules/features/graphicalEditor/mainGraphicalEditor.py similarity index 70% rename from modules/features/automation2/mainAutomation2.py rename to modules/features/graphicalEditor/mainGraphicalEditor.py index 80c8f854..e2cb5188 100644 --- a/modules/features/automation2/mainAutomation2.py +++ b/modules/features/graphicalEditor/mainGraphicalEditor.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Automation2 Feature - n8n-style flow automation. +GraphicalEditor Feature - n8n-style flow automation. Minimal bootstrap for feature instance creation. Build from here. """ @@ -10,9 +10,8 @@ from typing import Dict, List, Any, Optional logger = logging.getLogger(__name__) -FEATURE_CODE = "automation2" +FEATURE_CODE = "graphicalEditor" -# Services required for automation2 (methodDiscovery, ActionExecutor, etc.) REQUIRED_SERVICES = [ {"serviceKey": "chat", "meta": {"usage": "Interfaces, RBAC"}}, {"serviceKey": "utils", "meta": {"usage": "Timestamps, utilities"}}, @@ -22,82 +21,96 @@ REQUIRED_SERVICES = [ {"serviceKey": "clickup", "meta": {"usage": "ClickUp actions"}}, {"serviceKey": "generation", "meta": {"usage": "file.create document rendering"}}, ] -FEATURE_LABEL = {"en": "Automation 2", "de": "Automatisierung 2", "fr": "Automatisation 2"} +FEATURE_LABEL = {"en": "Graphical Editor", "de": "Grafischer Editor", "fr": "Éditeur graphique"} FEATURE_ICON = "mdi-sitemap" UI_OBJECTS = [ { - "objectKey": "ui.feature.automation2.editor", + "objectKey": "ui.feature.graphicalEditor.dashboard", + "label": {"en": "Dashboard", "de": "Dashboard", "fr": "Tableau de bord"}, + "meta": {"area": "dashboard"} + }, + { + "objectKey": "ui.feature.graphicalEditor.editor", "label": {"en": "Editor", "de": "Editor", "fr": "Éditeur"}, "meta": {"area": "editor"} }, { - "objectKey": "ui.feature.automation2.workflows", + "objectKey": "ui.feature.graphicalEditor.workflows", "label": {"en": "Workflows", "de": "Workflows", "fr": "Workflows"}, "meta": {"area": "workflows"} }, { - "objectKey": "ui.feature.automation2.workflows-tasks", + "objectKey": "ui.feature.graphicalEditor.workflows-tasks", "label": {"en": "Tasks", "de": "Tasks", "fr": "Tâches"}, "meta": {"area": "tasks"} }, + { + "objectKey": "ui.feature.graphicalEditor.templates", + "label": {"en": "Templates", "de": "Vorlagen", "fr": "Modèles"}, + "meta": {"area": "templates"} + }, ] RESOURCE_OBJECTS = [ { - "objectKey": "resource.feature.automation2.dashboard", + "objectKey": "resource.feature.graphicalEditor.dashboard", "label": {"en": "Access Dashboard", "de": "Dashboard aufrufen", "fr": "Acceder au tableau de bord"}, - "meta": {"endpoint": "/api/automation2/{instanceId}/info", "method": "GET"} + "meta": {"endpoint": "/api/workflows/{instanceId}/info", "method": "GET"} }, { - "objectKey": "resource.feature.automation2.node-types", + "objectKey": "resource.feature.graphicalEditor.node-types", "label": {"en": "Get Node Types", "de": "Node-Typen abrufen", "fr": "Obtenir types de nœuds"}, - "meta": {"endpoint": "/api/automation2/{instanceId}/node-types", "method": "GET"} + "meta": {"endpoint": "/api/workflows/{instanceId}/node-types", "method": "GET"} }, { - "objectKey": "resource.feature.automation2.execute", + "objectKey": "resource.feature.graphicalEditor.execute", "label": {"en": "Execute Workflow", "de": "Workflow ausführen", "fr": "Exécuter le workflow"}, - "meta": {"endpoint": "/api/automation2/{instanceId}/execute", "method": "POST"} + "meta": {"endpoint": "/api/workflows/{instanceId}/execute", "method": "POST"} }, ] TEMPLATE_ROLES = [ { - "roleLabel": "automation2-viewer", + "roleLabel": "graphicalEditor-viewer", "description": { - "en": "Automation2 Viewer - View workflows (read-only)", - "de": "Automation2 Betrachter - Workflows ansehen (nur lesen)", - "fr": "Visualiseur Automation2 - Consulter les workflows (lecture seule)", + "en": "GraphicalEditor Viewer - View workflows (read-only)", + "de": "Grafischer Editor Betrachter - Workflows ansehen (nur lesen)", + "fr": "Visualiseur Éditeur graphique - Consulter les workflows (lecture seule)", }, "accessRules": [ - {"context": "UI", "item": "ui.feature.automation2.workflows", "view": True}, - {"context": "UI", "item": "ui.feature.automation2.workflows-tasks", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.workflows", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.workflows-tasks", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.templates", "view": True}, {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, ], }, { - "roleLabel": "automation2-user", + "roleLabel": "graphicalEditor-user", "description": { - "en": "Automation2 User - Use automation2 flow builder", - "de": "Automation2 Benutzer - Flow-Builder nutzen", - "fr": "Utilisateur Automation2 - Utiliser le flow builder", + "en": "GraphicalEditor User - Use flow builder", + "de": "Grafischer Editor Benutzer - Flow-Builder nutzen", + "fr": "Utilisateur Éditeur graphique - Utiliser le flow builder", }, "accessRules": [ - {"context": "UI", "item": "ui.feature.automation2.editor", "view": True}, - {"context": "UI", "item": "ui.feature.automation2.workflows", "view": True}, - {"context": "UI", "item": "ui.feature.automation2.workflows-tasks", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.automation2.dashboard", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.automation2.node-types", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.automation2.execute", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.editor", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.workflows", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.workflows-tasks", "view": True}, + {"context": "UI", "item": "ui.feature.graphicalEditor.templates", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.graphicalEditor.dashboard", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.graphicalEditor.node-types", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.graphicalEditor.execute", "view": True}, {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, ], }, { - "roleLabel": "automation2-admin", + "roleLabel": "graphicalEditor-admin", "description": { - "en": "Automation2 Admin - Full UI and API for the instance; data remains user-scoped (MY)", - "de": "Automation2 Admin - Volle UI und API für die Instanz; Daten weiterhin benutzerspezifisch (MY)", - "fr": "Administrateur Automation2 - UI et API complets pour l'instance; donnees limitees a l'utilisateur (MY)", + "en": "GraphicalEditor Admin - Full UI and API for the instance; data remains user-scoped (MY)", + "de": "Grafischer Editor Admin - Volle UI und API für die Instanz; Daten weiterhin benutzerspezifisch (MY)", + "fr": "Administrateur Éditeur graphique - UI et API complets pour l'instance; donnees limitees a l'utilisateur (MY)", }, "accessRules": [ {"context": "UI", "item": None, "view": True}, @@ -113,14 +126,14 @@ def getRequiredServiceKeys() -> List[str]: return [s["serviceKey"] for s in REQUIRED_SERVICES] -def getAutomation2Services( +def getGraphicalEditorServices( user, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None, workflow=None, -) -> "_Automation2ServiceHub": +) -> "_GraphicalEditorServiceHub": """ - Get a service hub for automation2 using the service center. + Get a service hub for graphicalEditor using the service center. Used for methodDiscovery (I/O nodes) and execution (ActionExecutor). """ from modules.serviceCenter import getService @@ -141,7 +154,7 @@ def getAutomation2Services( workflow=_workflow, ) - hub = _Automation2ServiceHub() + hub = _GraphicalEditorServiceHub() hub.user = user hub.mandateId = mandateId hub.featureInstanceId = featureInstanceId @@ -155,7 +168,7 @@ def getAutomation2Services( svc = getService(key, ctx) setattr(hub, key, svc) except Exception as e: - logger.warning(f"Could not resolve service '{key}' for automation2: {e}") + logger.warning(f"Could not resolve service '{key}' for graphicalEditor: {e}") setattr(hub, key, None) if hub.chat: @@ -167,8 +180,12 @@ def getAutomation2Services( return hub -class _Automation2ServiceHub: - """Lightweight hub for automation2 (methodDiscovery, execution).""" +# Backward-compatible alias used by workflows/automation2/ execution engine +getAutomation2Services = getGraphicalEditorServices + + +class _GraphicalEditorServiceHub: + """Lightweight hub for graphicalEditor (methodDiscovery, execution).""" user = None mandateId = None @@ -190,12 +207,14 @@ class _Automation2ServiceHub: async def onStart(eventUser) -> None: - """Feature startup. Email poller is started on-demand when a run pauses for email.checkEmail.""" + """Feature startup: start consolidated scheduler.""" + from modules.workflows.scheduler.mainScheduler import start as startScheduler + startScheduler(eventUser) async def onStop(eventUser) -> None: """Feature shutdown - remove email poller if running.""" - from modules.features.automation2.emailPoller import stop as stopEmailPoller + from modules.features.graphicalEditor.emailPoller import stop as stopEmailPoller stopEmailPoller(eventUser) @@ -283,7 +302,6 @@ def _syncTemplateRolesToDb() -> int: _ensureAccessRulesForRole(rootInterface, roleId, template.get("accessRules", [])) - # Sync same rules to mandate-specific roles (so Workflows & Tasks etc. appear in sidebar) for r in existingRoles: if r.mandateId and r.roleLabel == roleLabel: added = _ensureAccessRulesForRole( diff --git a/modules/features/automation2/nodeDefinitions/__init__.py b/modules/features/graphicalEditor/nodeDefinitions/__init__.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/__init__.py rename to modules/features/graphicalEditor/nodeDefinitions/__init__.py diff --git a/modules/features/automation2/nodeDefinitions/ai.py b/modules/features/graphicalEditor/nodeDefinitions/ai.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/ai.py rename to modules/features/graphicalEditor/nodeDefinitions/ai.py diff --git a/modules/features/automation2/nodeDefinitions/clickup.py b/modules/features/graphicalEditor/nodeDefinitions/clickup.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/clickup.py rename to modules/features/graphicalEditor/nodeDefinitions/clickup.py diff --git a/modules/features/automation2/nodeDefinitions/email.py b/modules/features/graphicalEditor/nodeDefinitions/email.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/email.py rename to modules/features/graphicalEditor/nodeDefinitions/email.py diff --git a/modules/features/automation2/nodeDefinitions/file.py b/modules/features/graphicalEditor/nodeDefinitions/file.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/file.py rename to modules/features/graphicalEditor/nodeDefinitions/file.py diff --git a/modules/features/automation2/nodeDefinitions/flow.py b/modules/features/graphicalEditor/nodeDefinitions/flow.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/flow.py rename to modules/features/graphicalEditor/nodeDefinitions/flow.py diff --git a/modules/features/automation2/nodeDefinitions/input.py b/modules/features/graphicalEditor/nodeDefinitions/input.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/input.py rename to modules/features/graphicalEditor/nodeDefinitions/input.py diff --git a/modules/features/automation2/nodeDefinitions/sharepoint.py b/modules/features/graphicalEditor/nodeDefinitions/sharepoint.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/sharepoint.py rename to modules/features/graphicalEditor/nodeDefinitions/sharepoint.py diff --git a/modules/features/automation2/nodeDefinitions/triggers.py b/modules/features/graphicalEditor/nodeDefinitions/triggers.py similarity index 100% rename from modules/features/automation2/nodeDefinitions/triggers.py rename to modules/features/graphicalEditor/nodeDefinitions/triggers.py diff --git a/modules/features/automation2/nodeRegistry.py b/modules/features/graphicalEditor/nodeRegistry.py similarity index 94% rename from modules/features/automation2/nodeRegistry.py rename to modules/features/graphicalEditor/nodeRegistry.py index 4bcc9ba5..524b96af 100644 --- a/modules/features/automation2/nodeRegistry.py +++ b/modules/features/graphicalEditor/nodeRegistry.py @@ -1,14 +1,14 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Node Type Registry for automation2 - static node definitions (ai, email, sharepoint, trigger, flow, data, input). +Node Type Registry for graphicalEditor - static node definitions (ai, email, sharepoint, trigger, flow, data, input). Nodes are defined first; IO/method actions are used at execution time. """ import logging from typing import Dict, List, Any -from modules.features.automation2.nodeDefinitions import STATIC_NODE_TYPES +from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES logger = logging.getLogger(__name__) diff --git a/modules/features/automation2/routeFeatureAutomation2.py b/modules/features/graphicalEditor/routeFeatureGraphicalEditor.py similarity index 54% rename from modules/features/automation2/routeFeatureAutomation2.py rename to modules/features/graphicalEditor/routeFeatureGraphicalEditor.py index aa40f8bb..c2673a98 100644 --- a/modules/features/automation2/routeFeatureAutomation2.py +++ b/modules/features/graphicalEditor/routeFeatureGraphicalEditor.py @@ -1,26 +1,31 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Automation2 routes - node-types, execute, workflows, runs, tasks, connections, browse. +GraphicalEditor routes - node-types, execute, workflows, runs, tasks, connections, browse. """ +import asyncio +import json import logging -from typing import Any, Dict, Optional +import math +from typing import Any, Dict, List, Optional from fastapi import APIRouter, Depends, Path, Query, Body, Request, HTTPException -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from modules.auth import limiter, getRequestContext, RequestContext +from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict +from modules.routes.routeDataUsers import _applyFiltersAndSort -from modules.features.automation2.mainAutomation2 import getAutomation2Services -from modules.features.automation2.nodeRegistry import getNodeTypesForApi -from modules.features.automation2.interfaceFeatureAutomation2 import getAutomation2Interface +from modules.features.graphicalEditor.mainGraphicalEditor import getGraphicalEditorServices +from modules.features.graphicalEditor.nodeRegistry import getNodeTypesForApi +from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface from modules.workflows.automation2.executionEngine import executeGraph from modules.workflows.automation2.runEnvelope import ( default_run_envelope, merge_run_envelope, normalize_run_envelope, ) -from modules.features.automation2.entryPoints import find_invocation +from modules.features.graphicalEditor.entryPoints import find_invocation logger = logging.getLogger(__name__) @@ -85,14 +90,14 @@ def _build_execute_run_envelope( return env router = APIRouter( - prefix="/api/automation2", - tags=["Automation2"], + prefix="/api/workflows", + tags=["GraphicalEditor"], responses={404: {"description": "Not found"}, 403: {"description": "Forbidden"}}, ) def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: - """Validate user has access to the automation2 feature instance. Returns mandateId.""" + """Validate user has access to the graphicalEditor feature instance. Returns mandateId.""" from fastapi import HTTPException from modules.interfaces.interfaceDbApp import getRootInterface @@ -108,7 +113,7 @@ def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: @router.get("/{instanceId}/info") @limiter.limit("60/minute") -def get_automation2_info( +def get_info( request: Request, instanceId: str = Path(..., description="Feature instance ID"), context: RequestContext = Depends(getRequestContext), @@ -116,10 +121,10 @@ def get_automation2_info( """Minimal info endpoint - proves the feature works.""" _validateInstanceAccess(instanceId, context) return { - "featureCode": "automation2", + "featureCode": "graphicalEditor", "instanceId": instanceId, "status": "ok", - "message": "Automation2 feature ready. Build from here.", + "message": "GraphicalEditor feature ready.", } @@ -152,16 +157,16 @@ def get_node_types( context: RequestContext = Depends(getRequestContext), ) -> dict: """Return node types for the flow builder: static + I/O from methodDiscovery.""" - logger.info("automation2 node-types request: instanceId=%s language=%s", instanceId, language) + logger.info("graphicalEditor node-types request: instanceId=%s language=%s", instanceId, language) mandateId = _validateInstanceAccess(instanceId, context) - services = getAutomation2Services( + services = getGraphicalEditorServices( context.user, mandateId=mandateId, featureInstanceId=instanceId, ) result = getNodeTypesForApi(services, language=language) logger.info( - "automation2 node-types response: %d nodeTypes %d categories", + "graphicalEditor node-types response: %d nodeTypes %d categories", len(result.get("nodeTypes", [])), len(result.get("categories", [])), ) @@ -176,21 +181,20 @@ async def post_execute( body: dict = Body(..., description="{ workflowId?, graph: { nodes, connections } }"), context: RequestContext = Depends(getRequestContext), ) -> dict: - """Execute automation2 graph. Body: { workflowId?, graph: { nodes, connections } }.""" + """Execute workflow graph. Body: { workflowId?, graph: { nodes, connections } }.""" userId = str(context.user.id) if context.user else None logger.info( - "automation2 execute request: instanceId=%s userId=%s body_keys=%s", + "graphicalEditor execute request: instanceId=%s userId=%s body_keys=%s", instanceId, userId, list(body.keys()), ) mandateId = _validateInstanceAccess(instanceId, context) - services = getAutomation2Services( + services = getGraphicalEditorServices( context.user, mandateId=mandateId, featureInstanceId=instanceId, ) - # Ensure workflow methods (outlook, ai, sharepoint, etc.) are discovered for ActionExecutor from modules.workflows.processing.shared.methodDiscovery import discoverMethods discoverMethods(services) @@ -199,27 +203,23 @@ async def post_execute( req_nodes = graph.get("nodes") or [] workflow_for_envelope: Optional[Dict[str, Any]] = None if workflowId and not str(workflowId).startswith("transient-"): - a2_pre = getAutomation2Interface(context.user, mandateId, instanceId) - workflow_for_envelope = a2_pre.getWorkflow(workflowId) - # When workflowId is set: prefer graph from request (current editor state) if it has nodes. - # Only fall back to stored workflow graph when request graph is empty (e.g. resume from email). + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + workflow_for_envelope = iface.getWorkflow(workflowId) if workflowId and len(req_nodes) == 0: - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - wf = a2.getWorkflow(workflowId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) if wf and wf.get("graph"): graph = wf["graph"] - logger.info("automation2 execute: loaded graph from workflow %s", workflowId) + logger.info("graphicalEditor execute: loaded graph from workflow %s", workflowId) workflow_for_envelope = wf - # Use transient workflowId when none provided (e.g. execute from editor without save) - # Required for email.checkEmail pause/resume - run must be created if not workflowId: import uuid workflowId = f"transient-{uuid.uuid4().hex[:12]}" - logger.info("automation2 execute: using transient workflowId=%s", workflowId) + logger.info("graphicalEditor execute: using transient workflowId=%s", workflowId) nodes_count = len(graph.get("nodes") or []) connections_count = len(graph.get("connections") or []) logger.info( - "automation2 execute: graph nodes=%d connections=%d workflowId=%s mandateId=%s", + "graphicalEditor execute: graph nodes=%d connections=%d workflowId=%s mandateId=%s", nodes_count, connections_count, workflowId, @@ -227,7 +227,7 @@ async def post_execute( ) run_env = _build_execute_run_envelope(body, workflow_for_envelope, userId) - a2_interface = getAutomation2Interface(context.user, mandateId, instanceId) + ge_interface = getGraphicalEditorInterface(context.user, mandateId, instanceId) result = await executeGraph( graph=graph, services=services, @@ -235,11 +235,11 @@ async def post_execute( instanceId=instanceId, userId=userId, mandateId=mandateId, - automation2_interface=a2_interface, + automation2_interface=ge_interface, run_envelope=run_env, ) logger.info( - "automation2 execute result: success=%s error=%s nodeOutputs_keys=%s failedNode=%s paused=%s", + "graphicalEditor execute result: success=%s error=%s nodeOutputs_keys=%s failedNode=%s paused=%s", result.get("success"), result.get("error"), list(result.get("nodeOutputs", {}).keys()) if result.get("nodeOutputs") else [], @@ -250,7 +250,368 @@ async def post_execute( # ------------------------------------------------------------------------- -# Connections and Browse (for Email/SharePoint node config - like workspace) +# Versions (AutoVersion Lifecycle) +# ------------------------------------------------------------------------- + + +@router.get("/{instanceId}/workflows/{workflowId}/versions") +@limiter.limit("60/minute") +def get_versions( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + workflowId: str = Path(..., description="Workflow ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """List all versions for a workflow.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + versions = iface.getVersions(workflowId) + return {"versions": versions} + + +@router.post("/{instanceId}/workflows/{workflowId}/versions/draft") +@limiter.limit("30/minute") +def create_draft_version( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + workflowId: str = Path(..., description="Workflow ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Create a new draft version from the workflow's current graph.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + version = iface.createDraftVersion(workflowId) + if not version: + raise HTTPException(status_code=404, detail="Workflow not found") + return version + + +@router.post("/{instanceId}/versions/{versionId}/publish") +@limiter.limit("30/minute") +def publish_version( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + versionId: str = Path(..., description="Version ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Publish a draft version. Archives the previously published version.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + userId = str(context.user.id) if context.user else None + version = iface.publishVersion(versionId, userId=userId) + if not version: + raise HTTPException(status_code=400, detail="Version not found or not in draft status") + return version + + +@router.post("/{instanceId}/versions/{versionId}/unpublish") +@limiter.limit("30/minute") +def unpublish_version( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + versionId: str = Path(..., description="Version ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Unpublish a version (revert to draft).""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + version = iface.unpublishVersion(versionId) + if not version: + raise HTTPException(status_code=400, detail="Version not found or not published") + return version + + +@router.post("/{instanceId}/versions/{versionId}/archive") +@limiter.limit("30/minute") +def archive_version( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + versionId: str = Path(..., description="Version ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Archive a version.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + version = iface.archiveVersion(versionId) + if not version: + raise HTTPException(status_code=404, detail="Version not found") + return version + + +# ------------------------------------------------------------------------- +# Templates +# ------------------------------------------------------------------------- + + +@router.get("/{instanceId}/templates") +@limiter.limit("60/minute") +def get_templates( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + scope: Optional[str] = Query(None, description="Filter by scope: user, instance, mandate, system"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), + context: RequestContext = Depends(getRequestContext), +): + """List workflow templates with optional pagination.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + templates = iface.getTemplates(scope=scope) + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + if paginationParams: + filtered = _applyFiltersAndSort(templates, paginationParams) + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, pageSize=paginationParams.pageSize, + totalItems=totalItems, totalPages=totalPages, + sort=paginationParams.sort, filters=paginationParams.filters, + ).model_dump(), + } + return {"templates": templates} + + +@router.post("/{instanceId}/templates/from-workflow") +@limiter.limit("30/minute") +def create_template_from_workflow( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + body: dict = Body(..., description="{ workflowId, scope? }"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Create a template from an existing workflow.""" + mandateId = _validateInstanceAccess(instanceId, context) + workflowId = body.get("workflowId") + scope = body.get("scope", "user") + if not workflowId: + raise HTTPException(status_code=400, detail="workflowId required") + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + template = iface.createTemplateFromWorkflow(workflowId, scope=scope) + if not template: + raise HTTPException(status_code=404, detail="Workflow not found") + return template + + +@router.post("/{instanceId}/templates/{templateId}/copy") +@limiter.limit("30/minute") +def copy_template( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + templateId: str = Path(..., description="Template ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Copy a template to a new user-owned workflow.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + workflow = iface.copyTemplateToUser(templateId) + if not workflow: + raise HTTPException(status_code=404, detail="Template not found") + return workflow + + +@router.post("/{instanceId}/templates/{templateId}/share") +@limiter.limit("30/minute") +def share_template( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + templateId: str = Path(..., description="Template ID"), + body: dict = Body(..., description="{ scope }"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Share a template by changing its scope.""" + mandateId = _validateInstanceAccess(instanceId, context) + scope = body.get("scope") + if not scope or scope not in ("user", "instance", "mandate", "system"): + raise HTTPException(status_code=400, detail="scope must be user, instance, mandate, or system") + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + template = iface.shareTemplate(templateId, scope=scope) + if not template: + raise HTTPException(status_code=404, detail="Template not found") + return template + + +# ------------------------------------------------------------------------- +# AI Chat for Editor +# ------------------------------------------------------------------------- + + +@router.post("/{instanceId}/{workflowId}/chat/stream") +@limiter.limit("30/minute") +async def post_editor_chat( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + workflowId: str = Path(..., description="Workflow ID"), + body: dict = Body(..., description="{ message, conversationHistory?, userLanguage? }"), + context: RequestContext = Depends(getRequestContext), +): + """AI chat endpoint for the editor with SSE streaming. Uses workflow tools to mutate the graph.""" + mandateId = _validateInstanceAccess(instanceId, context) + message = body.get("message", "") + if not message: + raise HTTPException(status_code=400, detail="message required") + + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + raise HTTPException(status_code=404, detail="Workflow not found") + + userLanguage = body.get("userLanguage", "de") + conversationHistory = body.get("conversationHistory") or [] + + from modules.serviceCenter.core.serviceStreaming import get_event_manager + sseEventManager = get_event_manager() + queueId = f"ge-chat-{workflowId}-{id(request)}" + sseEventManager.create_queue(queueId) + + agentTask = asyncio.ensure_future( + _runEditorAgent( + workflowId=workflowId, + queueId=queueId, + prompt=message, + instanceId=instanceId, + user=context.user, + mandateId=mandateId, + sseEventManager=sseEventManager, + userLanguage=userLanguage, + conversationHistory=conversationHistory, + ) + ) + sseEventManager.register_agent_task(queueId, agentTask) + + async def _sseGenerator(): + queue = sseEventManager.get_queue(queueId) + if not queue: + return + while True: + try: + event = await asyncio.wait_for(queue.get(), timeout=120) + except asyncio.TimeoutError: + yield "data: {\"type\": \"keepalive\"}\n\n" + continue + if event is None: + break + ssePayload = event.get("data", event) if isinstance(event, dict) else event + yield f"data: {json.dumps(ssePayload, default=str)}\n\n" + eventType = ssePayload.get("type", "") if isinstance(ssePayload, dict) else "" + if eventType in ("complete", "error", "stopped"): + break + await sseEventManager.cleanup(queueId, delay=30) + + return StreamingResponse( + _sseGenerator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + +async def _runEditorAgent( + workflowId: str, + queueId: str, + prompt: str, + instanceId: str, + user=None, + mandateId: str = "", + sseEventManager=None, + userLanguage: str = "de", + conversationHistory: List[Dict[str, Any]] = None, +): + """Run the serviceAgent loop with workflow toolbox and forward events to the SSE queue.""" + try: + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum + + ctx = ServiceCenterContext( + user=user, + mandate_id=mandateId, + feature_instance_id=instanceId, + workflow_id=workflowId, + feature_code="graphicalEditor", + ) + agentService = getService("agent", ctx) + + systemPrompt = ( + "You are a workflow editor assistant. The user describes changes to a workflow graph. " + "Use the available workflow tools (readWorkflowGraph, addNode, removeNode, connectNodes, " + "setNodeParameter, listAvailableNodeTypes, validateGraph) to modify the graph. " + "Always read the current graph first before making changes. " + "Respond concisely and confirm what you changed." + ) + + accumulatedText = "" + + async for event in agentService.runAgent( + prompt=prompt, + workflowId=workflowId, + userLanguage=userLanguage, + conversationHistory=conversationHistory or [], + toolSet="core", + additionalTools=None, + systemPromptOverride=systemPrompt, + ): + if sseEventManager.is_cancelled(queueId): + logger.info("Editor chat agent cancelled for workflow %s", workflowId) + break + + if event.type == AgentEventTypeEnum.CHUNK and event.content: + accumulatedText += event.content + + sseEvent = { + "type": event.type.value if hasattr(event.type, "value") else event.type, + "workflowId": workflowId, + } + if event.content: + sseEvent["content"] = event.content + if event.data: + sseEvent["item"] = event.data + + await sseEventManager.emit_event(queueId, sseEvent["type"], sseEvent) + + if event.type in (AgentEventTypeEnum.FINAL, AgentEventTypeEnum.ERROR): + break + + await sseEventManager.emit_event(queueId, "complete", { + "type": "complete", + "workflowId": workflowId, + }) + + except asyncio.CancelledError: + logger.info("Editor chat agent task cancelled for workflow %s", workflowId) + await sseEventManager.emit_event(queueId, "stopped", { + "type": "stopped", + "workflowId": workflowId, + }) + + except Exception as e: + logger.error("Editor chat agent error: %s", e, exc_info=True) + await sseEventManager.emit_event(queueId, "error", { + "type": "error", + "content": str(e), + "workflowId": workflowId, + }) + finally: + sseEventManager._unregister_agent_task(queueId) + + +# ------------------------------------------------------------------------- +# Connections and Browse (for Email/SharePoint node config) # ------------------------------------------------------------------------- @@ -273,7 +634,7 @@ def _buildResolverDbInterface(chatService): @router.get("/{instanceId}/connections") @limiter.limit("300/minute") -def list_automation2_connections( +def list_connections( request: Request, instanceId: str = Path(..., description="Feature instance ID"), context: RequestContext = Depends(getRequestContext), @@ -441,20 +802,17 @@ def get_workflows( request: Request, instanceId: str = Path(..., description="Feature instance ID"), active: Optional[bool] = Query(None, description="Filter by active: true|false"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext), -) -> dict: - """List all workflows for this feature instance. - Enriches each workflow with runCount, isRunning, stuckAtNodeId, stuckAtNodeLabel, - createdAt, lastStartedAt. - Query param active: filter by active status (true|false). - """ +): + """List all workflows for this feature instance.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - items = a2.getWorkflows(active=active) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + items = iface.getWorkflows(active=active) enriched = [] for wf in items: wf_id = wf.get("id") - runs = a2.getRunsByWorkflow(wf_id) if wf_id else [] + runs = iface.getRunsByWorkflow(wf_id) if wf_id else [] run_count = len(runs) active_run = None last_started_at = None @@ -478,6 +836,31 @@ def get_workflows( "createdAt": wf.get("sysCreatedAt"), "lastStartedAt": last_started_at, }) + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + if paginationParams: + filtered = _applyFiltersAndSort(enriched, paginationParams) + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, pageSize=paginationParams.pageSize, + totalItems=totalItems, totalPages=totalPages, + sort=paginationParams.sort, filters=paginationParams.filters, + ).model_dump(), + } return {"workflows": enriched} @@ -491,8 +874,8 @@ def get_workflow( ) -> dict: """Get a single workflow by ID.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - wf = a2.getWorkflow(workflowId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) if not wf: raise HTTPException(status_code=404, detail="Workflow not found") return wf @@ -508,8 +891,8 @@ def create_workflow( ) -> dict: """Create a new workflow.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - created = a2.createWorkflow(body) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + created = iface.createWorkflow(body) return created @@ -524,8 +907,8 @@ def update_workflow( ) -> dict: """Update a workflow.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - updated = a2.updateWorkflow(workflowId, body) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + updated = iface.updateWorkflow(workflowId, body) if not updated: raise HTTPException(status_code=404, detail="Workflow not found") return updated @@ -541,8 +924,8 @@ def delete_workflow( ) -> dict: """Delete a workflow.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - if not a2.deleteWorkflow(workflowId): + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + if not iface.deleteWorkflow(workflowId): raise HTTPException(status_code=404, detail="Workflow not found") return {"success": True} @@ -557,14 +940,11 @@ async def post_workflow_webhook( body: dict = Body(default_factory=dict), context: RequestContext = Depends(getRequestContext), ) -> dict: - """ - Invoke a workflow via a webhook entry point. Optional shared secret in - X-Automation2-Webhook-Secret or X-Webhook-Secret when config.webhookSecret is set. - """ + """Invoke a workflow via a webhook entry point.""" mandateId = _validateInstanceAccess(instanceId, context) userId = str(context.user.id) if context.user else None - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - wf = a2.getWorkflow(workflowId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) if not wf or not wf.get("graph"): raise HTTPException(status_code=404, detail="Workflow not found") inv = find_invocation(wf, entryPointId) @@ -577,19 +957,16 @@ async def post_workflow_webhook( cfg = inv.get("config") or {} secret = cfg.get("webhookSecret") if secret: - hdr = request.headers.get("X-Automation2-Webhook-Secret") or request.headers.get( - "X-Webhook-Secret" - ) + hdr = request.headers.get("X-Webhook-Secret") if hdr != str(secret): raise HTTPException(status_code=403, detail="Invalid webhook secret") - services = getAutomation2Services( + services = getGraphicalEditorServices( context.user, mandateId=mandateId, featureInstanceId=instanceId, ) from modules.workflows.processing.shared.methodDiscovery import discoverMethods - discoverMethods(services) title = inv.get("title") or {} @@ -615,7 +992,7 @@ async def post_workflow_webhook( instanceId=instanceId, userId=userId, mandateId=mandateId, - automation2_interface=a2, + automation2_interface=iface, run_envelope=run_env, ) return result @@ -634,8 +1011,8 @@ async def post_workflow_form_submit( """Form-style submit: same as execute with trigger.type form and payload from body.""" mandateId = _validateInstanceAccess(instanceId, context) userId = str(context.user.id) if context.user else None - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - wf = a2.getWorkflow(workflowId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) if not wf or not wf.get("graph"): raise HTTPException(status_code=404, detail="Workflow not found") inv = find_invocation(wf, entryPointId) @@ -646,13 +1023,12 @@ async def post_workflow_form_submit( if not inv.get("enabled", True): raise HTTPException(status_code=400, detail="Entry point is disabled") - services = getAutomation2Services( + services = getGraphicalEditorServices( context.user, mandateId=mandateId, featureInstanceId=instanceId, ) from modules.workflows.processing.shared.methodDiscovery import discoverMethods - discoverMethods(services) title = inv.get("title") or {} @@ -678,7 +1054,7 @@ async def post_workflow_form_submit( instanceId=instanceId, userId=userId, mandateId=mandateId, - automation2_interface=a2, + automation2_interface=iface, run_envelope=run_env, ) return result @@ -697,10 +1073,10 @@ def get_completed_runs( limit: int = Query(20, ge=1, le=50), context: RequestContext = Depends(getRequestContext), ) -> dict: - """Get recently completed runs with output (for Tasks page output section).""" + """Get recently completed runs with output.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - runs = a2.getRecentCompletedRuns(limit=limit) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + runs = iface.getRecentCompletedRuns(limit=limit) return {"runs": runs} @@ -714,13 +1090,33 @@ def get_workflow_runs( ) -> dict: """Get runs for a workflow.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - if not a2.getWorkflow(workflowId): + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + if not iface.getWorkflow(workflowId): raise HTTPException(status_code=404, detail="Workflow not found") - runs = a2.getRunsByWorkflow(workflowId) + runs = iface.getRunsByWorkflow(workflowId) return {"runs": runs} +@router.get("/{instanceId}/runs/{runId}/steps") +@limiter.limit("60/minute") +def get_run_steps( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + runId: str = Path(..., description="Run ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Get step logs for a run (AutoStepLog entries).""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoStepLog + if not iface.db._ensureTableExists(AutoStepLog): + return {"steps": []} + records = iface.db.getRecordset(AutoStepLog, recordFilter={"runId": runId}) + steps = [dict(r) for r in records] if records else [] + steps.sort(key=lambda s: s.get("startedAt") or 0) + return {"steps": steps} + + @router.post("/{instanceId}/runs/{runId}/resume") @limiter.limit("30/minute") async def resume_run( @@ -732,32 +1128,29 @@ async def resume_run( ) -> dict: """Resume a paused run after task completion.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - run = a2.getRun(runId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + run = iface.getRun(runId) if not run: raise HTTPException(status_code=404, detail="Run not found") taskId = body.get("taskId") result = body.get("result") if not taskId or result is None: raise HTTPException(status_code=400, detail="taskId and result required") - task = a2.getTask(taskId) + task = iface.getTask(taskId) if not task or task.get("runId") != runId: raise HTTPException(status_code=404, detail="Task not found") if task.get("status") != "pending": raise HTTPException(status_code=400, detail="Task already completed") - a2.updateTask(taskId, status="completed", result=result) + iface.updateTask(taskId, status="completed", result=result) nodeId = task.get("nodeId") nodeOutputs = dict(run.get("nodeOutputs") or {}) nodeOutputs[nodeId] = result - runContext = run.get("context") or {} - connectionMap = runContext.get("connectionMap", {}) - inputSources = runContext.get("inputSources", {}) workflowId = run.get("workflowId") - wf = a2.getWorkflow(workflowId) if workflowId else None + wf = iface.getWorkflow(workflowId) if workflowId else None if not wf or not wf.get("graph"): raise HTTPException(status_code=400, detail="Workflow graph not found") graph = wf["graph"] - services = getAutomation2Services(context.user, mandateId=mandateId, featureInstanceId=instanceId) + services = getGraphicalEditorServices(context.user, mandateId=mandateId, featureInstanceId=instanceId) resume_result = await executeGraph( graph=graph, services=services, @@ -765,7 +1158,7 @@ async def resume_run( instanceId=instanceId, userId=str(context.user.id) if context.user else None, mandateId=mandateId, - automation2_interface=a2, + automation2_interface=iface, initialNodeOutputs=nodeOutputs, startAfterNodeId=nodeId, runId=runId, @@ -787,14 +1180,12 @@ def get_tasks( status: str = Query(None, description="Filter: pending, completed, rejected"), context: RequestContext = Depends(getRequestContext), ) -> dict: - """Get tasks - by default those assigned to current user, or all if no assignee filter. - Enriches each task with workflowLabel and createdAt (from sysCreatedAt). - """ + """Get tasks assigned to current user.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) assigneeId = str(context.user.id) if context.user else None - items = a2.getTasks(workflowId=workflowId, status=status, assigneeId=assigneeId) - workflows = {w["id"]: w for w in a2.getWorkflows()} + items = iface.getTasks(workflowId=workflowId, status=status, assigneeId=assigneeId) + workflows = {w["id"]: w for w in iface.getWorkflows()} enriched = [] for t in items: wf = workflows.get(t.get("workflowId") or "") @@ -817,29 +1208,29 @@ async def complete_task( ) -> dict: """Complete a task and resume the run.""" mandateId = _validateInstanceAccess(instanceId, context) - a2 = getAutomation2Interface(context.user, mandateId, instanceId) - task = a2.getTask(taskId) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + task = iface.getTask(taskId) if not task: raise HTTPException(status_code=404, detail="Task not found") runId = task.get("runId") result = body.get("result") if result is None: raise HTTPException(status_code=400, detail="result required") - run = a2.getRun(runId) + run = iface.getRun(runId) if not run: raise HTTPException(status_code=404, detail="Run not found") if task.get("status") != "pending": raise HTTPException(status_code=400, detail="Task already completed") - a2.updateTask(taskId, status="completed", result=result) + iface.updateTask(taskId, status="completed", result=result) nodeId = task.get("nodeId") nodeOutputs = dict(run.get("nodeOutputs") or {}) nodeOutputs[nodeId] = result workflowId = run.get("workflowId") - wf = a2.getWorkflow(workflowId) if workflowId else None + wf = iface.getWorkflow(workflowId) if workflowId else None if not wf or not wf.get("graph"): raise HTTPException(status_code=400, detail="Workflow graph not found") graph = wf["graph"] - services = getAutomation2Services(context.user, mandateId=mandateId, featureInstanceId=instanceId) + services = getGraphicalEditorServices(context.user, mandateId=mandateId, featureInstanceId=instanceId) return await executeGraph( graph=graph, services=services, @@ -847,8 +1238,64 @@ async def complete_task( instanceId=instanceId, userId=str(context.user.id) if context.user else None, mandateId=mandateId, - automation2_interface=a2, + automation2_interface=iface, initialNodeOutputs=nodeOutputs, startAfterNodeId=nodeId, runId=runId, ) + + +# ------------------------------------------------------------------------- +# Monitoring / Metrics +# ------------------------------------------------------------------------- + + +@router.get("/{instanceId}/metrics") +@limiter.limit("60/minute") +def get_metrics( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + context: RequestContext = Depends(getRequestContext), +) -> dict: + """Aggregated metrics for the monitoring dashboard.""" + mandateId = _validateInstanceAccess(instanceId, context) + iface = getGraphicalEditorInterface(context.user, mandateId, instanceId) + + from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import ( + AutoWorkflow, AutoRun, AutoStepLog, AutoTask, + ) + + workflows = iface.db.getRecordset(AutoWorkflow, recordFilter={ + "mandateId": mandateId, "featureInstanceId": instanceId, "isTemplate": False, + }) or [] + runs = iface.db.getRecordset(AutoRun, recordFilter={ + "workflowId": {"$in": [w.get("id") for w in workflows]} if workflows else "__none__", + }) or [] + tasks = iface.db.getRecordset(AutoTask, recordFilter={ + "workflowId": {"$in": [w.get("id") for w in workflows]} if workflows else "__none__", + }) or [] + + runsByStatus = {} + totalTokens = 0 + totalCredits = 0.0 + for r in runs: + s = r.get("status", "unknown") + runsByStatus[s] = runsByStatus.get(s, 0) + 1 + totalTokens += r.get("costTokens", 0) or 0 + totalCredits += r.get("costCredits", 0.0) or 0.0 + + tasksByStatus = {} + for t in tasks: + s = t.get("status", "unknown") + tasksByStatus[s] = tasksByStatus.get(s, 0) + 1 + + return { + "workflowCount": len(workflows), + "activeWorkflows": sum(1 for w in workflows if w.get("active")), + "totalRuns": len(runs), + "runsByStatus": runsByStatus, + "totalTasks": len(tasks), + "tasksByStatus": tasksByStatus, + "totalTokens": totalTokens, + "totalCredits": round(totalCredits, 4), + } diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index b3b8bcd0..e30d11ed 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -88,9 +88,6 @@ def initBootstrap(db: DatabaseConnector) -> None: # Apply multi-tenant database optimizations (indexes, triggers, FKs) _applyDatabaseOptimizations(db) - # Seed automation templates (after admin user exists) - initAutomationTemplates(db, adminUserId) - # Run root-user migration (one-time, sets completion flag) migrationDone = False try: @@ -148,86 +145,6 @@ def initBootstrap(db: DatabaseConnector) -> None: logger.warning(f"Mandate retention purge failed: {e}") -def initAutomationTemplates(dbApp: DatabaseConnector, adminUserId: Optional[str] = None) -> None: - """ - Seed initial automation templates from subAutomationTemplates.py. - Only runs if no templates exist yet (bootstrap). - Creates templates with sysCreatedBy = admin user (SysAdmin privilege). - - NOTE: AutomationTemplate lives in poweron_automation database, not poweron_app! - - Args: - dbApp: Database connector for poweron_app (used to get admin user if needed) - adminUserId: Admin user ID for sysCreatedBy field - """ - import json - from modules.features.automation.subAutomationTemplates import AUTOMATION_TEMPLATES - from modules.features.automation.datamodelFeatureAutomation import AutomationTemplate - from modules.shared.configuration import APP_CONFIG - - # Create connector for poweron_automation database (where templates live) - dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") - dbDatabase = "poweron_automation" - dbUser = APP_CONFIG.get("DB_USER") - dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) - - dbAutomation = DatabaseConnector( - dbHost=dbHost, - dbDatabase=dbDatabase, - dbUser=dbUser, - dbPassword=dbPassword, - dbPort=dbPort, - userId=adminUserId, - ) - dbAutomation.initDbSystem() - - # Check if templates already exist in poweron_automation - existing = dbAutomation.getRecordset(AutomationTemplate) - if existing: - logger.info(f"Automation templates already seeded ({len(existing)} templates)") - return - - # Get admin user ID if not provided (from poweron_app) - if not adminUserId: - adminUsers = dbApp.getRecordset(UserInDB, recordFilter={"email": APP_CONFIG.ADMIN_EMAIL}) - adminUserId = adminUsers[0]["id"] if adminUsers else None - # Update context with admin user - if adminUserId: - dbAutomation.updateContext(adminUserId) - - templates = AUTOMATION_TEMPLATES.get("sets", []) - createdCount = 0 - - for i, templateSet in enumerate(templates): - templateContent = templateSet.get("template", {}) - overview = templateContent.get("overview", f"Template {i+1}") - - # Create multilingual label from overview (use as German since current templates are German) - # English is required by TextMultilingual, so we use the same value - labelDict = {"en": overview, "ge": overview} - overviewDict = {"en": overview, "ge": overview} - - # Create template WITHOUT parameters (no sharp values) - templateData = { - "label": labelDict, - "overview": overviewDict, - "template": json.dumps(templateContent), # Store entire template JSON - "isSystem": True, # Seeded templates are system-level, visible to all users - } - - try: - dbAutomation.recordCreate(AutomationTemplate, templateData) - createdCount += 1 - logger.debug(f"Created automation template: {overview}") - except Exception as e: - logger.error(f"Failed to create automation template '{overview}': {e}") - - logger.info(f"Seeded {createdCount} automation templates in poweron_automation database") - - logger.info("System bootstrap completed") - - def initRootMandateFeatures(db: DatabaseConnector, mandateId: str) -> None: """ Create feature instances for root mandate. @@ -517,7 +434,7 @@ def initRoles(db: DatabaseConnector) -> None: # Check specifically for system template roles: # mandateId=NULL, isSystemRole=True, featureCode=NULL - # Feature templates (e.g. automation admin) share the same labels but have featureCode set! + # Feature templates share the same labels but have featureCode set! allTemplates = db.getRecordset( Role, recordFilter={"mandateId": None, "isSystemRole": True} @@ -549,7 +466,7 @@ def _deduplicateRoles(db: DatabaseConnector) -> None: # Group by (roleLabel, mandateId, featureInstanceId, featureCode) # featureCode is essential: system template ('admin', None, None, None) - # must NOT be grouped with feature template ('admin', None, None, 'automation') + # must NOT be grouped with feature template ('admin', None, None, '') groups: dict = {} for role in allRoles: key = (role.get("roleLabel"), role.get("mandateId"), role.get("featureInstanceId"), role.get("featureCode")) @@ -986,12 +903,11 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: # - data.uam.* → User Access Management (mandantenübergreifend) # - data.chat.* → Chat/AI-Daten (benutzer-eigen, kein Mandantenkontext) # - data.files.* → Dateien (benutzer-eigen) - # - data.automation.* → Automation (benutzer-eigen) # - data.feature.* → Mandanten-/Feature-spezifische Daten (dynamisch) # # GROUP-Berechtigung: # - data.uam.*: GROUP filtert nach Mandant (via UserMandate) - # - data.chat.*, data.files.*, data.automation.*: GROUP = MY (benutzer-eigen) + # - data.chat.*, data.files.*: GROUP = MY (benutzer-eigen) # ========================================================================== # ------------------------------------------------------------------------- @@ -1248,70 +1164,6 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # ------------------------------------------------------------------------- - # Automation Namespace - User-owned, no mandate context - # ------------------------------------------------------------------------- - - # AutomationDefinition: Only MY-level access (user-owned) - for roleId in [adminId, userId]: - if roleId: - tableRules.append(AccessRule( - roleId=roleId, - context=AccessRuleContext.DATA, - item="data.automation.AutomationDefinition", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - if viewerId: - tableRules.append(AccessRule( - roleId=viewerId, - context=AccessRuleContext.DATA, - item="data.automation.AutomationDefinition", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # AutomationTemplate: Admin sees ALL (system templates), User sees only MY - if adminId: - tableRules.append(AccessRule( - roleId=adminId, - context=AccessRuleContext.DATA, - item="data.automation.AutomationTemplate", - view=True, - read=AccessLevel.ALL, # SysAdmin sees all templates - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - if userId: - tableRules.append(AccessRule( - roleId=userId, - context=AccessRuleContext.DATA, - item="data.automation.AutomationTemplate", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - if viewerId: - tableRules.append(AccessRule( - roleId=viewerId, - context=AccessRuleContext.DATA, - item="data.automation.AutomationTemplate", - view=True, - read=AccessLevel.ALL, # Viewer can see all templates (read-only) - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - # ------------------------------------------------------------------------- # Billing Namespace - Billing accounts and transactions # ------------------------------------------------------------------------- @@ -1623,12 +1475,6 @@ def _ensureDataContextRules(db: DatabaseConnector) -> None: # Users can only manage their own records (MY-level access) tablesNeedingMyRules = [ "data.chat.ChatWorkflow", - "data.automation.AutomationDefinition", - ] - - # Tables where admin sees ALL (system-wide templates) - tablesNeedingAllRulesForAdmin = [ - "data.automation.AutomationTemplate", ] # Billing tables: read-only for all roles, scoped by role level @@ -1681,47 +1527,6 @@ def _ensureDataContextRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # Admin rules for system templates (read ALL, write GROUP-scoped) - for objectKey in tablesNeedingAllRulesForAdmin: - # Admin: read ALL templates, create/update/delete within GROUP (mandate-scoped) - if adminId and (adminId, objectKey) not in existingCombinations: - missingRules.append(AccessRule( - roleId=adminId, - context=AccessRuleContext.DATA, - item=objectKey, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - - # User: MY-level access - if userId and (userId, objectKey) not in existingCombinations: - missingRules.append(AccessRule( - roleId=userId, - context=AccessRuleContext.DATA, - item=objectKey, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - - # Viewer: ALL read-only (can see all templates) - if viewerId and (viewerId, objectKey) not in existingCombinations: - missingRules.append(AccessRule( - roleId=viewerId, - context=AccessRuleContext.DATA, - item=objectKey, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - # Billing read-only rules: Admin=GROUP, User/Viewer=MY (own accounts/transactions) for objectKey in billingReadOnlyTables: # Admin: GROUP-level read (sees all accounts in their mandates) @@ -1806,59 +1611,6 @@ def _ensureDataContextRules(db: DatabaseConnector) -> None: logger.info(f"Created {len(missingRules)} missing DATA context rules") # All DATA context rules already exist (nothing to create) - # Update existing AutomationTemplate rules for admin/viewer to ALL access - _updateAutomationTemplateRulesToAll(db, adminId, viewerId) - - -def _updateAutomationTemplateRulesToAll(db: DatabaseConnector, adminId: Optional[str], viewerId: Optional[str]) -> None: - """ - Update existing AutomationTemplate RBAC rules to correct levels. - - Admin: read=ALL, create/update/delete=GROUP (mandate-scoped writes) - - Viewer: read=ALL (read-only) - """ - if not adminId and not viewerId: - return - - templateObjectKey = "data.automation.AutomationTemplate" - - # Find existing rules for AutomationTemplate - existingRules = db.getRecordset( - AccessRule, - recordFilter={ - "context": AccessRuleContext.DATA.value, - "item": templateObjectKey - } - ) - - updatedCount = 0 - for rule in existingRules: - ruleId = rule.get("id") - roleId = rule.get("roleId") - currentReadLevel = rule.get("read") - - if roleId == adminId: - # Admin: read ALL, write GROUP - updates = {} - if currentReadLevel != AccessLevel.ALL.value: - updates["read"] = AccessLevel.ALL.value - currentCreate = rule.get("create") - if currentCreate == AccessLevel.ALL.value: - updates["create"] = AccessLevel.GROUP.value - updates["update"] = AccessLevel.GROUP.value - updates["delete"] = AccessLevel.GROUP.value - if updates: - db.recordModify(AccessRule, ruleId, updates) - updatedCount += 1 - logger.debug(f"Updated AutomationTemplate rule {ruleId} for admin to read=ALL, write=GROUP") - - elif roleId == viewerId and currentReadLevel == AccessLevel.MY.value: - # Viewer: read ALL (read-only) - db.recordModify(AccessRule, ruleId, {"read": AccessLevel.ALL.value}) - updatedCount += 1 - logger.debug(f"Updated AutomationTemplate rule {ruleId} for viewer to read=ALL") - - if updatedCount > 0: - logger.info(f"Updated {updatedCount} AutomationTemplate RBAC rules") def _createResourceContextRules(db: DatabaseConnector) -> None: @@ -2002,7 +1754,6 @@ def _createStoreResourceRules(db: DatabaseConnector) -> None: db: Database connector instance """ storeResources = [ - "resource.store.automation", "resource.store.teamsbot", "resource.store.workspace", "resource.store.commcoach", diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 947a6e2d..e93a12a5 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -74,10 +74,16 @@ TABLE_NAMESPACE = { # Automation - benutzer-eigen "AutomationDefinition": "automation", "AutomationTemplate": "automation", - # Automation2 - feature-scoped - "Automation2Workflow": "automation2", - "Automation2WorkflowRun": "automation2", - "Automation2HumanTask": "automation2", + # GraphicalEditor - Greenfield DB poweron_graphicaleditor (Auto-prefix models) + "AutoWorkflow": "feature.graphicalEditor", + "AutoVersion": "feature.graphicalEditor", + "AutoRun": "feature.graphicalEditor", + "AutoStepLog": "feature.graphicalEditor", + "AutoTask": "feature.graphicalEditor", + # Legacy aliases (backward compat) + "Automation2Workflow": "feature.graphicalEditor", + "Automation2WorkflowRun": "feature.graphicalEditor", + "Automation2HumanTask": "feature.graphicalEditor", # Knowledge Store - benutzer-eigen "FileContentIndex": "knowledge", "ContentChunk": "knowledge", diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py deleted file mode 100644 index 553c66d3..00000000 --- a/modules/routes/routeAdminAutomationEvents.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Admin automation events routes for the backend API. -Sysadmin-only endpoints for viewing and controlling scheduler events. -""" - -from fastapi import APIRouter, HTTPException, Depends, Path, Request, Response, Query -from typing import List, Dict, Any, Optional -from fastapi import status -import logging -import json -import math - -# Import interfaces and models from feature containers -import modules.features.automation.interfaceFeatureAutomation as interfaceAutomation -from modules.auth import limiter, getRequestContext, requireSysAdminRole, RequestContext -from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict -from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues - -# Configure logger -logger = logging.getLogger(__name__) - -# Create router for admin automation events endpoints -router = APIRouter( - prefix="/api/admin/automation-events", - tags=["Admin Automation Events"], - responses={ - 404: {"description": "Not found"}, - 400: {"description": "Bad request"}, - 401: {"description": "Unauthorized"}, - 403: {"description": "Forbidden - Sysadmin only"}, - 500: {"description": "Internal server error"} - } -) - -def _buildEnrichedAutomationEvents(currentUser: User) -> List[Dict[str, Any]]: - """Build the full enriched automation events list.""" - from modules.shared.eventManagement import eventManager - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.features.automation.mainAutomation import getAutomationServices - - if not eventManager.scheduler: - return [] - - jobs = [] - for job in eventManager.scheduler.get_jobs(): - if job.id.startswith("automation."): - automationId = job.id.replace("automation.", "") - jobs.append({ - "eventId": job.id, - "id": job.id, - "automationId": automationId, - "nextRunTime": str(job.next_run_time) if job.next_run_time else None, - "trigger": str(job.trigger) if job.trigger else None, - "name": "", - "createdBy": "", - "mandate": "", - "featureInstance": "" - }) - - if jobs: - try: - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") - if eventUser: - services = getAutomationServices(currentUser, mandateId=None, featureInstanceId=None) - allAutomations = services.interfaceDbAutomation.getAllAutomationDefinitionsWithRBAC(eventUser) - - automationLookup = {} - for a in allAutomations: - aId = a.get("id", "") if isinstance(a, dict) else getattr(a, "id", "") - automationLookup[aId] = a - - _userCache: Dict[str, str] = {} - _mandateCache: Dict[str, str] = {} - _featureCache: Dict[str, str] = {} - - def _resolveUsername(userId): - if not userId: return "" - if userId not in _userCache: - try: - user = rootInterface.getUser(userId) - _userCache[userId] = user.username if user else userId[:8] - except Exception: - _userCache[userId] = userId[:8] - return _userCache[userId] - - def _resolveMandateLabel(mandateId): - if not mandateId: return "" - if mandateId not in _mandateCache: - try: - mandate = rootInterface.getMandate(mandateId) - _mandateCache[mandateId] = getattr(mandate, "label", None) or mandateId[:8] - except Exception: - _mandateCache[mandateId] = mandateId[:8] - return _mandateCache[mandateId] - - def _resolveFeatureLabel(featureInstanceId): - if not featureInstanceId: return "" - if featureInstanceId not in _featureCache: - try: - instance = rootInterface.getFeatureInstance(featureInstanceId) - _featureCache[featureInstanceId] = getattr(instance, "label", None) or getattr(instance, "featureCode", None) or featureInstanceId[:8] - except Exception: - _featureCache[featureInstanceId] = featureInstanceId[:8] - return _featureCache[featureInstanceId] - - for job in jobs: - automation = automationLookup.get(job["automationId"]) - if automation: - if isinstance(automation, dict): - job["name"] = automation.get("label", "") - job["createdBy"] = _resolveUsername(automation.get("sysCreatedBy", "")) - job["mandate"] = _resolveMandateLabel(automation.get("mandateId", "")) - job["featureInstance"] = _resolveFeatureLabel(automation.get("featureInstanceId", "")) - else: - job["name"] = getattr(automation, "label", "") - job["createdBy"] = _resolveUsername(getattr(automation, "sysCreatedBy", "")) - job["mandate"] = _resolveMandateLabel(getattr(automation, "mandateId", "")) - job["featureInstance"] = _resolveFeatureLabel(getattr(automation, "featureInstanceId", "")) - else: - job["name"] = "(orphaned)" - except Exception as e: - logger.warning(f"Could not enrich automation events with context: {e}") - - return jobs - - -@router.get("") -@limiter.limit("30/minute") -def get_all_automation_events( - request: Request, - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), - currentUser: User = Depends(requireSysAdminRole), -): - """Get all active scheduler jobs with pagination support (sysadmin only).""" - try: - paginationParams: Optional[PaginationParams] = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") - - enriched = _buildEnrichedAutomationEvents(currentUser) - filtered = _applyFiltersAndSort(enriched, paginationParams) - - if paginationParams: - totalItems = len(filtered) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - return { - "items": filtered[startIdx:endIdx], - "pagination": PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters, - ).model_dump(), - } - - return {"items": enriched, "pagination": None} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting automation events: {str(e)}") - raise HTTPException(status_code=500, detail=f"Error getting automation events: {str(e)}") - - -@router.get("/filter-values") -@limiter.limit("60/minute") -def get_automation_event_filter_values( - request: Request, - column: str = Query(..., description="Column key"), - pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), - currentUser: User = Depends(requireSysAdminRole), -): - """Return distinct filter values for a column in automation events.""" - try: - crossFilterParams: Optional[PaginationParams] = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - filters = paginationDict.get("filters", {}) - filters.pop(column, None) - paginationDict["filters"] = filters - paginationDict.pop("sort", None) - crossFilterParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError): - pass - - enriched = _buildEnrichedAutomationEvents(currentUser) - crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) - return _extractDistinctValues(crossFiltered, column) - except Exception as e: - logger.error(f"Error getting filter values: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) - -@router.post("/sync") -@limiter.limit("5/minute") -async def sync_all_automation_events( - request: Request, - currentUser: User = Depends(requireSysAdminRole) -) -> Dict[str, Any]: - """ - Manually trigger sync for all automations (sysadmin only). - This will register/remove events based on active flags. - """ - try: - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.workflows.automation import syncAutomationEvents - - # Get event user for sync operation (routes can import from interfaces) - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") - if not eventUser: - raise HTTPException( - status_code=500, - detail="Event user not available" - ) - - from modules.features.automation.mainAutomation import getAutomationServices - services = getAutomationServices(currentUser, mandateId=None, featureInstanceId=None) - result = syncAutomationEvents(services, eventUser) - return { - "success": True, - "synced": result.get("synced", 0), - "events": result.get("events", {}) - } - except HTTPException: - raise - except Exception as e: - logger.error(f"Error syncing automation events: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error syncing automation events: {str(e)}" - ) - -@router.post("/{eventId}/remove") -@limiter.limit("10/minute") -def remove_event( - request: Request, - eventId: str = Path(..., description="Event ID to remove"), - currentUser: User = Depends(requireSysAdminRole) -) -> Dict[str, Any]: - """ - Remove a scheduler job (sysadmin only). - Removes the job from the scheduler and clears the eventId on the automation definition. - Does NOT delete the automation definition itself. - """ - try: - from modules.shared.eventManagement import eventManager - - # Remove scheduler job - eventManager.remove(eventId) - - # Clear eventId on the automation definition (so it can be re-synced later) - if eventId.startswith("automation."): - automationId = eventId.replace("automation.", "") - automationInterface = interfaceAutomation.getInterface(currentUser) - automation = automationInterface.getAutomationDefinition(automationId) - if automation and getattr(automation, "eventId", None) == eventId: - automationInterface.updateAutomationDefinition(automationId, {"eventId": None}) - - return { - "success": True, - "eventId": eventId, - "message": f"Event {eventId} removed successfully" - } - except Exception as e: - logger.error(f"Error removing event: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error removing event: {str(e)}" - ) diff --git a/modules/routes/routeAdminAutomationLogs.py b/modules/routes/routeAdminAutomationLogs.py deleted file mode 100644 index 479d0df3..00000000 --- a/modules/routes/routeAdminAutomationLogs.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Admin automation execution logs routes. -SysAdmin-only endpoints for viewing consolidated automation execution history -across all mandates and feature instances. -""" - -from fastapi import APIRouter, HTTPException, Depends, Request, Query -from typing import List, Dict, Any, Optional -import logging -import json -import math -import uuid - -from modules.auth import limiter, requireSysAdminRole -from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict -from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues - -logger = logging.getLogger(__name__) - -router = APIRouter( - prefix="/api/admin/automation-logs", - tags=["Admin Automation Logs"], - responses={ - 401: {"description": "Unauthorized"}, - 403: {"description": "Forbidden - Sysadmin only"}, - 500: {"description": "Internal server error"}, - }, -) - - -def _buildFlattenedExecutionLogs(currentUser: User) -> List[Dict[str, Any]]: - """Flatten executionLogs from all AutomationDefinitions across all mandates. - Called from a SysAdmin-only endpoint — bypasses RBAC, reads directly from DB.""" - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.features.automation.mainAutomation import getAutomationServices - from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition - - rootInterface = getRootInterface() - services = getAutomationServices(currentUser, mandateId=None, featureInstanceId=None) - allAutomations = services.interfaceDbAutomation.db.getRecordset(AutomationDefinition) - - userCache: Dict[str, str] = {} - mandateCache: Dict[str, str] = {} - featureCache: Dict[str, str] = {} - - def _resolveUsername(userId: str) -> str: - if not userId: - return "" - if userId not in userCache: - try: - user = rootInterface.getUser(userId) - userCache[userId] = user.username if user else userId[:8] - except Exception: - userCache[userId] = userId[:8] - return userCache[userId] - - def _resolveMandateLabel(mandateId: str) -> str: - if not mandateId: - return "" - if mandateId not in mandateCache: - try: - mandate = rootInterface.getMandate(mandateId) - mandateCache[mandateId] = getattr(mandate, "label", None) or mandateId[:8] - except Exception: - mandateCache[mandateId] = mandateId[:8] - return mandateCache[mandateId] - - def _resolveFeatureLabel(featureInstanceId: str) -> str: - if not featureInstanceId: - return "" - if featureInstanceId not in featureCache: - try: - instance = rootInterface.getFeatureInstance(featureInstanceId) - featureCache[featureInstanceId] = ( - getattr(instance, "label", None) - or getattr(instance, "featureCode", None) - or featureInstanceId[:8] - ) - except Exception: - featureCache[featureInstanceId] = featureInstanceId[:8] - return featureCache[featureInstanceId] - - flatLogs: List[Dict[str, Any]] = [] - - for automation in allAutomations: - if isinstance(automation, dict): - automationId = automation.get("id", "") - automationLabel = automation.get("label", "") - mandateId = automation.get("mandateId", "") - featureInstanceId = automation.get("featureInstanceId", "") - createdBy = automation.get("sysCreatedBy", "") - logs = automation.get("executionLogs") or [] - else: - automationId = getattr(automation, "id", "") - automationLabel = getattr(automation, "label", "") - mandateId = getattr(automation, "mandateId", "") - featureInstanceId = getattr(automation, "featureInstanceId", "") - createdBy = getattr(automation, "sysCreatedBy", "") - logs = getattr(automation, "executionLogs", None) or [] - - mandateName = _resolveMandateLabel(mandateId) - featureInstanceName = _resolveFeatureLabel(featureInstanceId) - executedByName = _resolveUsername(createdBy) - - for log in logs: - timestamp = log.get("timestamp", 0) if isinstance(log, dict) else 0 - status = log.get("status", "") if isinstance(log, dict) else "" - workflowId = log.get("workflowId", "") if isinstance(log, dict) else "" - messages = log.get("messages", []) if isinstance(log, dict) else [] - - flatLogs.append({ - "id": str(uuid.uuid4()), - "timestamp": timestamp, - "automationId": automationId, - "automationLabel": automationLabel, - "mandateName": mandateName, - "featureInstanceName": featureInstanceName, - "executedBy": executedByName, - "status": status, - "workflowId": workflowId, - "messages": "; ".join(messages) if messages else "", - }) - - flatLogs.sort(key=lambda x: x.get("timestamp", 0), reverse=True) - return flatLogs - - -@router.get("") -@limiter.limit("30/minute") -def get_all_automation_logs( - request: Request, - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), - currentUser: User = Depends(requireSysAdminRole), -): - """Get consolidated execution logs from all automations (sysadmin only).""" - try: - paginationParams: Optional[PaginationParams] = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") - - logs = _buildFlattenedExecutionLogs(currentUser) - filtered = _applyFiltersAndSort(logs, paginationParams) - - if paginationParams: - totalItems = len(filtered) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - return { - "items": filtered[startIdx:endIdx], - "pagination": PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters, - ).model_dump(), - } - - return {"items": logs, "pagination": None} - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting automation logs: {str(e)}") - raise HTTPException(status_code=500, detail=f"Error getting automation logs: {str(e)}") - - -@router.get("/filter-values") -@limiter.limit("60/minute") -def get_automation_log_filter_values( - request: Request, - column: str = Query(..., description="Column key"), - pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), - currentUser: User = Depends(requireSysAdminRole), -): - """Return distinct filter values for a column in automation logs.""" - try: - crossFilterParams: Optional[PaginationParams] = None - if pagination: - try: - paginationDict = json.loads(pagination) - if paginationDict: - paginationDict = normalize_pagination_dict(paginationDict) - filters = paginationDict.get("filters", {}) - filters.pop(column, None) - paginationDict["filters"] = filters - paginationDict.pop("sort", None) - crossFilterParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError): - pass - - logs = _buildFlattenedExecutionLogs(currentUser) - crossFiltered = _applyFiltersAndSort(logs, crossFilterParams) - return _extractDistinctValues(crossFiltered, column) - except Exception as e: - logger.error(f"Error getting filter values: {str(e)}") - raise HTTPException(status_code=500, detail=str(e)) diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 9d05daf6..d7f6b7a9 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -397,20 +397,23 @@ def create_feature( # Feature Instance Endpoints (Mandate-scoped) # ============================================================================= -@router.get("/instances", response_model=List[Dict[str, Any]]) +@router.get("/instances") @limiter.limit("60/minute") def list_feature_instances( request: Request, featureCode: Optional[str] = Query(None, description="Filter by feature code"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext) -) -> List[Dict[str, Any]]: +): """ List feature instances for the current mandate. Returns instances the user has access to within the selected mandate. + Supports server-side pagination, filtering, sorting, and search. Args: featureCode: Optional filter by feature code + pagination: JSON-encoded PaginationParams (page, pageSize, sort, filters) """ if not context.mandateId: raise HTTPException( @@ -419,6 +422,16 @@ def list_feature_instances( ) try: + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + rootInterface = getRootInterface() featureInterface = getFeatureInterface(rootInterface.db) @@ -427,7 +440,27 @@ def list_feature_instances( featureCode=featureCode ) - return [inst.model_dump() for inst in instances] + items = [inst.model_dump() for inst in instances] + + if paginationParams: + filtered = _applyFiltersAndSort(items, paginationParams) + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters, + ).model_dump(), + } + else: + return items except HTTPException: raise @@ -995,13 +1028,14 @@ class FeatureInstanceUserUpdate(BaseModel): enabled: Optional[bool] = Field(None, description="Whether this user's access is active (omit to leave unchanged)") -@router.get("/instances/{instanceId}/users", response_model=List[FeatureInstanceUserResponse]) +@router.get("/instances/{instanceId}/users") @limiter.limit("60/minute") def list_feature_instance_users( request: Request, instanceId: str, + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext) -) -> List[FeatureInstanceUserResponse]: +): """ List all users with access to a specific feature instance. @@ -1061,7 +1095,33 @@ def list_feature_instance_users( enabled=fa.enabled )) - return result + items = [r.model_dump() for r in result] + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + if paginationParams: + filtered = _applyFiltersAndSort(items, paginationParams) + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, pageSize=paginationParams.pageSize, + totalItems=totalItems, totalPages=totalPages, + sort=paginationParams.sort, filters=paginationParams.filters, + ).model_dump(), + } + return items except HTTPException: raise diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index 6e34eb88..43d803e3 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -14,10 +14,14 @@ from fastapi import APIRouter, HTTPException, Depends, Request, Query from typing import List, Dict, Any, Optional from fastapi import status import logging +import json +import math from pydantic import BaseModel, Field, model_validator from modules.auth import limiter, getRequestContext, RequestContext, getCurrentUser from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict +from modules.routes.routeDataUsers import _applyFiltersAndSort from modules.datamodels.datamodelInvitation import Invitation from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp @@ -394,15 +398,16 @@ def create_invitation( ) -@router.get("/", response_model=List[Dict[str, Any]]) +@router.get("/") @limiter.limit("60/minute") def list_invitations( request: Request, frontendUrl: str = Query(..., description="Frontend URL for building invite links (provided by frontend)"), includeUsed: bool = Query(False, description="Include already used invitations"), includeExpired: bool = Query(False, description="Include expired invitations"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext) -) -> List[Dict[str, Any]]: +): """ List invitations for the current mandate. @@ -468,6 +473,30 @@ def list_invitations( "isUsedUp": currentUses >= maxUses }) + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + if paginationParams: + filtered = _applyFiltersAndSort(result, paginationParams) + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, pageSize=paginationParams.pageSize, + totalItems=totalItems, totalPages=totalPages, + sort=paginationParams.sort, filters=paginationParams.filters, + ).model_dump(), + } return result except HTTPException: diff --git a/modules/routes/routeSystem.py b/modules/routes/routeSystem.py index f287d908..7e452be3 100644 --- a/modules/routes/routeSystem.py +++ b/modules/routes/routeSystem.py @@ -102,11 +102,8 @@ def _getFeatureUiObjects(featureCode: str) -> List[Dict[str, Any]]: elif featureCode == "realestate": from modules.features.realEstate.mainRealEstate import UI_OBJECTS return UI_OBJECTS - elif featureCode == "automation": - from modules.features.automation.mainAutomation import UI_OBJECTS - return UI_OBJECTS - elif featureCode == "automation2": - from modules.features.automation2.mainAutomation2 import UI_OBJECTS + elif featureCode == "graphicalEditor": + from modules.features.graphicalEditor.mainGraphicalEditor import UI_OBJECTS return UI_OBJECTS elif featureCode == "teamsbot": from modules.features.teamsbot.mainTeamsbot import UI_OBJECTS diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/__init__.py b/modules/serviceCenter/services/serviceAgent/coreTools/__init__.py new file mode 100644 index 00000000..e476ac39 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Core agent tools: registration of built-in ToolRegistry handlers.""" + +from .registerCore import registerCoreTools + +__all__ = ["registerCoreTools"] diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_connectionTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_connectionTools.py new file mode 100644 index 00000000..3419c8f8 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_connectionTools.py @@ -0,0 +1,193 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""External connection tools (list connections, upload, send mail).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerConnectionTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- Connection tools (external data sources) ---- + + def _buildResolverDb(): + """Build a DB adapter that ConnectorResolver can use to load UserConnections. + interfaceDbApp has getUserConnectionById; ConnectorResolver expects getUserConnection.""" + chatService = services.chat + appIf = getattr(chatService, "interfaceDbApp", None) + if appIf and hasattr(appIf, "getUserConnectionById"): + class _Adapter: + def __init__(self, app): + self._app = app + def getUserConnection(self, connectionId: str): + return self._app.getUserConnectionById(connectionId) + return _Adapter(appIf) + return getattr(chatService, "interfaceDbComponent", None) + + async def _listConnections(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + connections = chatService.getUserConnections() if hasattr(chatService, "getUserConnections") else [] + if not connections: + return ToolResult(toolCallId="", toolName="listConnections", success=True, data="No connections available.") + lines = [] + for conn in connections: + connId = conn.get("id", "?") if isinstance(conn, dict) else getattr(conn, "id", "?") + authority = conn.get("authority", "?") if isinstance(conn, dict) else getattr(conn, "authority", "?") + email = conn.get("externalEmail", "") if isinstance(conn, dict) else getattr(conn, "externalEmail", "") + lines.append(f"- {authority} ({email}) id: {connId}") + return ToolResult(toolCallId="", toolName="listConnections", success=True, data="\n".join(lines)) + except Exception as e: + return ToolResult(toolCallId="", toolName="listConnections", success=False, error=str(e)) + + async def _uploadToExternal(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + service = args.get("service", "") + path = args.get("path", "") + fileId = args.get("fileId", "") + if not connectionId or not service or not path or not fileId: + return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error="connectionId, service, path, and fileId are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + chatService = services.chat + fileContent = chatService.getFileContent(fileId) + if not fileContent: + return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error="File not found") + fileData = fileContent.get("data", b"") if isinstance(fileContent, dict) else b"" + if isinstance(fileData, str): + fileData = fileData.encode("utf-8") + fileName = fileContent.get("fileName", "file") if isinstance(fileContent, dict) else "file" + result = await adapter.upload(path, fileData, fileName) + return ToolResult(toolCallId="", toolName="uploadToExternal", success=True, data=str(result)) + except Exception as e: + return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error=str(e)) + + async def _sendMail(args: Dict[str, Any], context: Dict[str, Any]): + import base64 as _b64 + + connectionId = args.get("connectionId", "") + to = args.get("to", []) + subject = args.get("subject", "") + body = args.get("body", "") + bodyType = "HTML" if args.get("bodyType", "text").lower() == "html" else "Text" + draft = args.get("draft", False) + attachmentFileIds = args.get("attachmentFileIds") or [] + + if not connectionId or not to or not subject: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error="connectionId, to, and subject are required") + try: + graphAttachments: List[Dict[str, Any]] = [] + if attachmentFileIds: + chatService = services.chat + dbMgmt = chatService.interfaceDbComponent + for fid in attachmentFileIds: + fileRow = dbMgmt.getFile(fid) + if not fileRow: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error=f"Attachment file not found: {fid}") + rawBytes = dbMgmt.getFileData(fid) + if not rawBytes: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error=f"Attachment file has no data: {fid}") + graphAttachments.append({ + "name": fileRow.fileName, + "contentBytes": _b64.b64encode(rawBytes).decode("ascii"), + "contentType": getattr(fileRow, "mimeType", "application/octet-stream"), + }) + + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, "outlook") + + if draft and hasattr(adapter, "createDraft"): + result = await adapter.createDraft( + to=to, subject=subject, body=body, bodyType=bodyType, + cc=args.get("cc"), attachments=graphAttachments or None, + ) + return ToolResult(toolCallId="", toolName="sendMail", success=True, data=str(result)) + + if hasattr(adapter, "sendMail"): + result = await adapter.sendMail( + to=to, subject=subject, body=body, bodyType=bodyType, + cc=args.get("cc"), attachments=graphAttachments or None, + ) + return ToolResult(toolCallId="", toolName="sendMail", success=True, data=str(result)) + return ToolResult(toolCallId="", toolName="sendMail", success=False, error="Mail not supported by this adapter") + except Exception as e: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error=str(e)) + + _connToolParams = { + "connectionId": {"type": "string", "description": "UserConnection ID"}, + "service": {"type": "string", "description": "Service name (sharepoint, outlook, drive, etc.)"}, + } + + registry.register( + "listConnections", _listConnections, + description="List the user's external connections (SharePoint, OneDrive, Outlook, etc.) and their IDs. Use with browseDataSource/uploadToExternal.", + parameters={"type": "object", "properties": {}}, + readOnly=True, + ) + + registry.register( + "uploadToExternal", _uploadToExternal, + description=( + "Upload a local file to an external storage via connectionId+service. " + "Use listConnections to find available connections." + ), + parameters={ + "type": "object", + "properties": { + **_connToolParams, + "path": {"type": "string", "description": "Destination path on the external service"}, + "fileId": {"type": "string", "description": "Local file ID to upload"}, + }, + "required": ["connectionId", "service", "path", "fileId"], + }, + readOnly=False, + ) + + registry.register( + "sendMail", _sendMail, + description=( + "Send or draft an email via a connected mail service (Outlook). " + "Supports HTML body and file attachments from the workspace. " + "Set draft=true to save as draft without sending. " + "Use listConnections to find the connectionId." + ), + parameters={ + "type": "object", + "properties": { + "connectionId": {"type": "string", "description": "UserConnection ID"}, + "to": {"type": "array", "items": {"type": "string"}, "description": "Recipient email addresses"}, + "subject": {"type": "string", "description": "Email subject"}, + "body": {"type": "string", "description": "Email body — plain text or HTML markup"}, + "bodyType": {"type": "string", "enum": ["text", "html"], "description": "Body format: 'text' (default) or 'html'"}, + "cc": {"type": "array", "items": {"type": "string"}, "description": "CC addresses"}, + "attachmentFileIds": { + "type": "array", "items": {"type": "string"}, + "description": "File IDs from the workspace to attach (use listFiles to find IDs)", + }, + "draft": {"type": "boolean", "description": "If true, save as draft in Drafts folder instead of sending"}, + }, + "required": ["connectionId", "to", "subject", "body"], + }, + readOnly=False, + ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_crossWorkflowTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_crossWorkflowTools.py new file mode 100644 index 00000000..7307e019 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_crossWorkflowTools.py @@ -0,0 +1,175 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Cross-workflow tools and core-only tool-set tagging.""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerCrossWorkflowTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- Cross-workflow tools ---- + + async def _listWorkflowHistory(args: Dict[str, Any], context: Dict[str, Any]) -> ToolResult: + """List all chat workflows in this workspace with metadata.""" + import json as _json + try: + chatService = services.chat + chatInterface = chatService.interfaceDbChat + allWorkflows = chatInterface.getWorkflows() or [] + + allWorkflows.sort( + key=lambda w: w.get("sysCreatedAt") or w.get("startedAt") or 0, + reverse=True, + ) + allWorkflows = allWorkflows[:50] + + items = [] + for wf in allWorkflows: + wfId = wf.get("id", "") + name = wf.get("name") or "(unnamed)" + createdAt = wf.get("sysCreatedAt") or wf.get("startedAt") or 0 + lastActivity = wf.get("lastActivity") or createdAt + + msgs = chatInterface.getMessages(wfId) or [] + messageCount = len(msgs) + lastPreview = "" + if msgs: + lastMsg = msgs[-1] if isinstance(msgs[-1], dict) else ( + msgs[-1].model_dump() if hasattr(msgs[-1], "model_dump") else {} + ) + content = lastMsg.get("message") or lastMsg.get("content") or "" + lastPreview = content[:150] + + items.append({ + "id": wfId, + "name": name, + "createdAt": createdAt, + "lastActivity": lastActivity, + "messageCount": messageCount, + "lastMessagePreview": lastPreview, + }) + + return ToolResult( + toolCallId="", toolName="listWorkflowHistory", + success=True, data=_json.dumps(items, ensure_ascii=False), + ) + except Exception as e: + return ToolResult( + toolCallId="", toolName="listWorkflowHistory", + success=False, error=str(e), + ) + + registry.register( + "listWorkflowHistory", _listWorkflowHistory, + description=( + "List all chat conversations/workflows in this workspace. " + "Returns id, name, createdAt, lastActivity, messageCount, and a preview " + "of the last message for each workflow. Use this to discover previous " + "conversations when the user asks about past chats or wants a summary " + "across conversations." + ), + parameters={ + "type": "object", + "properties": {}, + }, + readOnly=True, + ) + + async def _readWorkflowMessages(args: Dict[str, Any], context: Dict[str, Any]) -> ToolResult: + """Read messages from a specific workflow.""" + import json as _json + targetWorkflowId = args.get("workflowId", "") + limit = int(args.get("limit", 20)) + offset = int(args.get("offset", 0)) + + if not targetWorkflowId: + return ToolResult( + toolCallId="", toolName="readWorkflowMessages", + success=False, error="workflowId is required", + ) + + try: + chatService = services.chat + chatInterface = chatService.interfaceDbChat + allMsgs = chatInterface.getMessages(targetWorkflowId) or [] + + sliced = allMsgs[offset:offset + limit] + items = [] + for msg in sliced: + raw = msg if isinstance(msg, dict) else ( + msg.model_dump() if hasattr(msg, "model_dump") else {} + ) + content = raw.get("message") or raw.get("content") or "" + if len(content) > 2000: + content = content[:2000] + "..." + items.append({ + "role": raw.get("role", ""), + "message": content, + "publishedAt": raw.get("publishedAt") or raw.get("sysCreatedAt") or 0, + }) + + header = f"Workflow {targetWorkflowId}: {len(allMsgs)} total messages" + if offset > 0 or len(allMsgs) > offset + limit: + header += f" (showing {offset + 1}-{offset + len(sliced)})" + + return ToolResult( + toolCallId="", toolName="readWorkflowMessages", + success=True, + data=header + "\n" + _json.dumps(items, ensure_ascii=False), + ) + except Exception as e: + return ToolResult( + toolCallId="", toolName="readWorkflowMessages", + success=False, error=str(e), + ) + + registry.register( + "readWorkflowMessages", _readWorkflowMessages, + description=( + "Read messages from a specific chat workflow/conversation. " + "Use this after listWorkflowHistory to read the content of a " + "specific past conversation. Supports pagination via offset/limit." + ), + parameters={ + "type": "object", + "properties": { + "workflowId": {"type": "string", "description": "ID of the workflow to read messages from"}, + "limit": {"type": "integer", "description": "Max messages to return (default 20)"}, + "offset": {"type": "integer", "description": "Skip first N messages (default 0)"}, + }, + "required": ["workflowId"], + }, + readOnly=True, + ) + + # Tag core-only tools so restricted toolSets (e.g. "commcoach") exclude them. + # Tools NOT in this set remain toolSet=None → available to ALL sets. + _CORE_ONLY_TOOLS = { + "listFiles", "listFolders", "tagFile", "moveFile", "createFolder", + "writeFile", "deleteFile", "renameFile", "translateText", + "deleteFolder", "renameFolder", "moveFolder", "copyFile", "replaceInFile", + "listConnections", "uploadToExternal", "sendMail", "downloadFromDataSource", + "browseContainer", "readContentObjects", "extractContainerItem", + "summarizeContent", "describeImage", "renderDocument", + "textToSpeech", "generateImage", "createChart", + "speechToText", "detectLanguage", "neutralizeData", "executeCode", + "listWorkflowHistory", "readWorkflowMessages", + } + for _toolName in _CORE_ONLY_TOOLS: + _td = registry.getTool(_toolName) + if _td: + _td.toolSet = "core" + diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py new file mode 100644 index 00000000..de50275d --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py @@ -0,0 +1,258 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""DataSource convenience tools (browse, search, download from external sources).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerDataSourceTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- DataSource convenience tools ---- + _SOURCE_TYPE_TO_SERVICE = { + "sharepointFolder": "sharepoint", + "onedriveFolder": "onedrive", + "outlookFolder": "outlook", + "googleDriveFolder": "drive", + "gmailFolder": "gmail", + "ftpFolder": "files", + "clickupList": "clickup", + } + + async def _resolveDataSource(dsId: str): + """Resolve a DataSource record and return (connectionId, service, path, neutralize) or raise.""" + chatService = services.chat + ds = chatService.getDataSource(dsId) if hasattr(chatService, "getDataSource") else None + if not ds: + raise ValueError(f"DataSource '{dsId}' not found") + connectionId = ds.get("connectionId", "") + sourceType = ds.get("sourceType", "") + path = ds.get("path", "/") + label = ds.get("label", "") + neutralize = bool(ds.get("neutralize", False)) + service = _SOURCE_TYPE_TO_SERVICE.get(sourceType, sourceType) + if not connectionId: + raise ValueError(f"DataSource '{dsId}' has no connectionId") + logger.info(f"Resolved DataSource '{dsId}' ({label}): sourceType={sourceType}, service={service}, connectionId={connectionId}, path={path[:80]}, neutralize={neutralize}") + return connectionId, service, path, neutralize + + _MAIL_SERVICES = {"outlook", "gmail"} + + async def _browseDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + subPath = args.get("subPath", "") + directConnId = args.get("connectionId", "") + directService = args.get("service", "") + if not dsId and not (directConnId and directService): + return ToolResult(toolCallId="", toolName="browseDataSource", success=False, + error="Provide either dataSourceId OR connectionId+service") + try: + if dsId: + connectionId, service, basePath, _neutralize = await _resolveDataSource(dsId) + else: + connectionId, service, basePath = directConnId, directService, args.get("path", "/") + if subPath: + if subPath.startswith("/"): + browsePath = subPath + else: + browsePath = f"{basePath.rstrip('/')}/{subPath}" + else: + browsePath = basePath + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.browse(browsePath, filter=args.get("filter")) + if not entries: + return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data="Empty directory.") + lines = [] + for e in entries: + prefix = "[DIR]" if e.isFolder else "[FILE]" + sizeInfo = f" ({e.size} bytes)" if e.size else "" + lines.append(f"- {prefix} {e.name}{sizeInfo} path: {e.path}") + result = "\n".join(lines) + if service in _MAIL_SERVICES: + result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." + return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="browseDataSource", success=False, error=str(e)) + + async def _searchDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + directConnId = args.get("connectionId", "") + directService = args.get("service", "") + query = args.get("query", "") + if not query: + return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error="query is required") + if not dsId and not (directConnId and directService): + return ToolResult(toolCallId="", toolName="searchDataSource", success=False, + error="Provide either dataSourceId OR connectionId+service") + try: + if dsId: + connectionId, service, basePath, _neutralize = await _resolveDataSource(dsId) + else: + connectionId, service, basePath = directConnId, directService, args.get("path", "/") + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.search(query, path=basePath) + if not entries: + return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data="No results found.") + lines = [f"- {e.name} (path: {e.path})" for e in entries] + result = "\n".join(lines) + if service in _MAIL_SERVICES: + result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." + return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error=str(e)) + + async def _downloadFromDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + directConnId = args.get("connectionId", "") + directService = args.get("service", "") + filePath = args.get("filePath", "") + fileName = args.get("fileName", "") + if not filePath: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="filePath is required") + if not dsId and not (directConnId and directService): + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, + error="Provide either dataSourceId OR connectionId+service") + try: + from modules.connectors.connectorResolver import ConnectorResolver + from modules.connectors.connectorProviderBase import DownloadResult as _DR + _sourceNeutralize = False + if dsId: + connectionId, service, basePath, _sourceNeutralize = await _resolveDataSource(dsId) + else: + connectionId, service, basePath = directConnId, directService, "/" + fullPath = filePath if filePath.startswith("/") else f"{basePath.rstrip('/')}/{filePath}" + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + result = await adapter.download(fullPath) + + if isinstance(result, _DR): + fileBytes = result.data + fileName = result.fileName or fileName + else: + fileBytes = result + + if not fileBytes: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="Download returned empty") + + if not fileName or "." not in fileName: + pathSegment = fullPath.split("/")[-1] or "downloaded_file" + fileName = fileName or pathSegment + if "." not in fileName: + try: + entries = await adapter.browse(basePath) + for entry in entries: + if getattr(entry, "path", "") == filePath or getattr(entry, "path", "").endswith(filePath): + if "." in entry.name: + fileName = entry.name + break + except Exception: + pass + if "." not in fileName: + if fileBytes[:4] == b"%PDF": + fileName = f"{fileName}.pdf" + elif fileBytes[:2] == b"PK": + fileName = f"{fileName}.zip" + chatService = services.chat + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(fileBytes, fileName) + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fileItem.id, {"featureInstanceId": fiId}) + if _sourceNeutralize: + chatService.interfaceDbComponent.updateFile(fileItem.id, {"neutralize": True}) + tempFolderId = _getOrCreateTempFolder(chatService) + if tempFolderId: + chatService.interfaceDbComponent.updateFile(fileItem.id, {"folderId": tempFolderId}) + ext = fileName.rsplit(".", 1)[-1].lower() if "." in fileName else "" + hint = "Use readFile to read the text content." if ext in ("doc", "docx", "txt", "csv", "json", "xml", "html", "md", "rtf", "odt", "xls", "xlsx", "pptx", "pdf", "eml", "msg") else "Use readFile to access the content." + return ToolResult( + toolCallId="", toolName="downloadFromDataSource", success=True, + data=f"Downloaded '{fileName}' ({len(fileBytes)} bytes) → local file id: {fileItem.id}. {hint}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error=str(e)) + + registry.register( + "browseDataSource", _browseDataSource, + description=( + "Browse files and folders in a data source. Accepts either:\n" + "- dataSourceId (for attached data sources shown in the prompt), OR\n" + "- connectionId + service (for direct connection access via listConnections)." + ), + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID (from attached data sources)"}, + "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, + "service": {"type": "string", "description": "Service name (alternative to dataSourceId, e.g. sharepoint, onedrive)"}, + "path": {"type": "string", "description": "Root path (used with connectionId+service)"}, + "subPath": {"type": "string", "description": "Sub-path within the data source to browse"}, + "filter": {"type": "string", "description": "Filter pattern (e.g. '*.pdf')"}, + }, + }, + readOnly=True, + ) + + registry.register( + "searchDataSource", _searchDataSource, + description=( + "Search for files within a data source. Accepts either dataSourceId OR connectionId+service." + ), + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID"}, + "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, + "service": {"type": "string", "description": "Service name (alternative to dataSourceId)"}, + "path": {"type": "string", "description": "Scope path (used with connectionId+service)"}, + "query": {"type": "string", "description": "Search query"}, + }, + "required": ["query"], + }, + readOnly=True, + ) + + registry.register( + "downloadFromDataSource", _downloadFromDataSource, + description=( + "Download a file or email from a data source into local storage. Returns a local file ID " + "to read with readFile. Accepts either dataSourceId OR connectionId+service. " + "For email sources (Outlook, Gmail), browse/search only return subjects -- use this to get full content." + ), + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID"}, + "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, + "service": {"type": "string", "description": "Service name (alternative to dataSourceId)"}, + "filePath": {"type": "string", "description": "Path of the file to download (from browseDataSource results)"}, + "fileName": {"type": "string", "description": "File name with extension (e.g. 'report.pdf')"}, + }, + "required": ["dataSourceId", "filePath"], + }, + readOnly=False, + ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_documentTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_documentTools.py new file mode 100644 index 00000000..b9c113b1 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_documentTools.py @@ -0,0 +1,374 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Document and vision tools (containers, content objects, image description).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerDocumentTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- Document tools (Smart Documents / Container Handling) ---- + + async def _browseContainer(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="browseContainer", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + index = knowledgeService.getFileContentIndex(fileId) + if not index: + return ToolResult(toolCallId="", toolName="browseContainer", success=True, data="No content index available for this file. It may not have been indexed yet.") + structure = index.get("structure", {}) if isinstance(index, dict) else {} + objectSummary = index.get("objectSummary", []) if isinstance(index, dict) else [] + totalObjects = index.get("totalObjects", 0) if isinstance(index, dict) else 0 + + result = f"File: {index.get('fileName', '?')} ({index.get('mimeType', '?')})\n" + result += f"Total content objects: {totalObjects}\n" + + sections = structure.get("sections", []) + if sections: + result += "\nSections:\n" + for s in sections: + result += f" [{s.get('id', '?')}] {s.get('title', 'Untitled')} (pages {s.get('startPage', '?')}-{s.get('endPage', '?')})\n" + + if structure.get("pageMap"): + pages = len(structure["pageMap"]) + result += f"\nPages: {pages}\n" + imgCount = structure.get("imageCount", 0) + tableCount = structure.get("tableCount", 0) + if imgCount: + result += f"Images: {imgCount}\n" + if tableCount: + result += f"Tables: {tableCount}\n" + + if structure.get("sheetMap"): + result += "\nSheets:\n" + for s in structure["sheetMap"]: + result += f" {s.get('sheetName', '?')} ({s.get('rows', '?')} rows x {s.get('columns', '?')} cols)\n" + + if structure.get("slideMap"): + result += "\nSlides:\n" + for s in structure["slideMap"]: + result += f" Slide {s.get('slideIndex', 0) + 1}: {s.get('title', '(no title)')}\n" + + return ToolResult(toolCallId="", toolName="browseContainer", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="browseContainer", success=False, error=str(e)) + + async def _readContentObjects(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + filterDict = {} + if args.get("pageIndex") is not None: + filterDict["pageIndex"] = args["pageIndex"] + if args.get("contentType"): + filterDict["contentType"] = args["contentType"] + if args.get("sectionId"): + filterDict["sectionId"] = args["sectionId"] + + objects = await knowledgeService.readContentObjects(fileId, filterDict) + if not objects: + return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data="No content objects found with the given filter.") + + result = f"Found {len(objects)} content objects:\n\n" + for obj in objects[:20]: + data = obj.get("data", "") + cType = obj.get("contentType", "?") + ref = obj.get("contextRef", {}) + location = ref.get("location", "") if isinstance(ref, dict) else "" + preview = data[:300] if cType == "text" else f"[{cType} data, {len(data)} chars]" + result += f"[{cType}] {location}: {preview}\n\n" + + if len(objects) > 20: + result += f"... and {len(objects) - 20} more objects" + + return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error=str(e)) + + async def _extractContainerItem(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + containerPath = args.get("containerPath", "") + if not fileId or not containerPath: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error="fileId and containerPath are required") + try: + knowledgeService = services.getService("knowledge") + result = await knowledgeService.extractContainerItem(fileId, containerPath) + if result: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=True, data=str(result)) + return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error=f"Item '{containerPath}' not found in container index for file {fileId}. On-demand extraction is not yet implemented.") + except Exception as e: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error=str(e)) + + async def _summarizeContent(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + filterDict = {} + if args.get("sectionId"): + filterDict["sectionId"] = args["sectionId"] + if args.get("pageIndex") is not None: + filterDict["pageIndex"] = args["pageIndex"] + if args.get("contentType"): + filterDict["contentType"] = args["contentType"] + + objects = await knowledgeService.readContentObjects(fileId, filterDict) + if not objects: + return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data="No content found to summarize.") + + textParts = [obj.get("data", "") for obj in objects if obj.get("contentType") != "image"] + combinedText = "\n\n".join(textParts)[:6000] + + aiService = services.ai + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + summaryRequest = AiCallRequest( + prompt=f"Summarize the following content concisely:\n\n{combinedText}", + options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE), + ) + response = await aiService.callAi(summaryRequest) + return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data=response.content) + except Exception as e: + return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error=str(e)) + + registry.register( + "browseContainer", _browseContainer, + description="Browse the structural index of a document (pages, sections, sheets, slides). Use before readContentObjects for targeted reading.", + parameters={ + "type": "object", + "properties": {"fileId": {"type": "string", "description": "The file ID to browse"}}, + "required": ["fileId"], + }, + readOnly=True, + ) + + registry.register( + "readContentObjects", _readContentObjects, + description="Read extracted content objects from a file, optionally filtered by page, section, or type. Use browseContainer first to see the structure.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "pageIndex": {"type": "integer", "description": "Filter by page index"}, + "sectionId": {"type": "string", "description": "Filter by section ID"}, + "contentType": {"type": "string", "description": "Filter by content type (text, image, etc.)"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) + + registry.register( + "extractContainerItem", _extractContainerItem, + description="Extract a specific item from a container file (ZIP, nested file). Use browseContainer to see available items.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The container file ID"}, + "containerPath": {"type": "string", "description": "Path within the container"}, + }, + "required": ["fileId", "containerPath"], + }, + readOnly=True, + ) + + registry.register( + "summarizeContent", _summarizeContent, + description="Generate an AI-powered summary of a file's content. Optionally filter by section, page, or content type.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "sectionId": {"type": "string", "description": "Optional: summarize only this section"}, + "pageIndex": {"type": "integer", "description": "Optional: summarize only this page"}, + "contentType": {"type": "string", "description": "Optional: filter by content type"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) + + # ---- Vision tool ---- + + async def _describeImage(args: Dict[str, Any], context: Dict[str, Any]): + """Analyse an image using AI vision. Uses Knowledge Store chunks produced by Extractors.""" + fileId = args.get("fileId", "") + prompt = args.get("prompt", "Describe this image in detail. Extract all visible text, tables, and data.") + pageIndex = args.get("pageIndex") + + if not fileId: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error="fileId is required") + + try: + import base64 as _b64 + + imageData = None + mimeType = "image/png" + + knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None + + # 1) Knowledge Store: image chunks already produced by PdfExtractor / ImageExtractor + if knowledgeService: + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if pageIndex is not None: + imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] + if imageChunks: + imageData = imageChunks[0].get("data", "") + chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType") + if chunkMime: + mimeType = chunkMime + + # 2) File not yet indexed -> trigger extraction via ExtractionService, then retry + if not imageData and knowledgeService and not knowledgeService.isFileIndexed(fileId): + try: + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) + fileContent = chatService.getFileContent(fileId) + if fileContent and fileInfo: + rawData = fileContent.get("data", "") + if isinstance(rawData, str) and len(rawData) > 100: + rawBytes = _b64.b64decode(rawData) + elif isinstance(rawData, bytes): + rawBytes = rawData + else: + rawBytes = None + + if rawBytes: + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction + from modules.datamodels.datamodelExtraction import ExtractionOptions + + fileMime = fileInfo.get("mimeType", "application/octet-stream") + fileName = fileInfo.get("fileName", fileId) + extracted = runExtraction( + ExtractorRegistry(), None, + rawBytes, fileName, fileMime, ExtractionOptions(), + ) + + contentObjects = [] + for part in extracted.parts: + tg = (part.typeGroup or "").lower() + ct = "image" if tg == "image" else "text" + if not part.data or not part.data.strip(): + continue + contentObjects.append({ + "contentObjectId": part.id, + "contentType": ct, + "data": part.data, + "contextRef": {"containerPath": fileName, "location": part.label, **(part.metadata or {})}, + }) + + if contentObjects: + _diFiId, _diMId = _resolveFileScope(fileId, context) + await knowledgeService.indexFile( + fileId=fileId, fileName=fileName, mimeType=fileMime, + userId=context.get("userId", ""), contentObjects=contentObjects, + featureInstanceId=_diFiId, + mandateId=_diMId, + ) + + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if pageIndex is not None: + imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] + if imageChunks: + imageData = imageChunks[0].get("data", "") + except Exception as extractErr: + logger.warning(f"describeImage: on-demand extraction failed: {extractErr}") + + # 3) Direct image file (not a container) - use raw file data + if not imageData: + chatService = services.chat + fileContent = chatService.getFileContent(fileId) + if fileContent: + fileMimeType = fileContent.get("mimeType", "") + if fileMimeType.startswith("image/"): + imageData = fileContent.get("data", "") + mimeType = fileMimeType + + if not imageData: + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) if hasattr(chatService, "getFileInfo") else None + fileName = fileInfo.get("fileName", fileId) if fileInfo else fileId + fileMime = fileInfo.get("mimeType", "unknown") if fileInfo else "unknown" + return ToolResult(toolCallId="", toolName="describeImage", success=False, + error=f"No image data found in '{fileName}' (type: {fileMime}). " + f"This file likely contains text, not images. Use readFile(fileId=\"{fileId}\") to access its text content.") + + try: + rawHead = _b64.b64decode(imageData[:32]) + if rawHead[:3] == b"\xff\xd8\xff": + mimeType = "image/jpeg" + elif rawHead[:8] == b"\x89PNG\r\n\x1a\n": + mimeType = "image/png" + elif rawHead[:4] == b"GIF8": + mimeType = "image/gif" + elif rawHead[:4] == b"RIFF" and rawHead[8:12] == b"WEBP": + mimeType = "image/webp" + except Exception: + pass + dataUrl = f"data:{mimeType};base64,{imageData}" + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum as OTE + + _opType = OTE.IMAGE_ANALYSE + try: + from modules.datamodels.datamodelFiles import FileItem as _FileItemModel + from modules.interfaces.interfaceDbManagement import ComponentObjects as _CO + _fRow = _CO().db._loadRecord(_FileItemModel, fileId) + if _fRow: + _fGet = (lambda k, d=None: _fRow.get(k, d)) if isinstance(_fRow, dict) else (lambda k, d=None: getattr(_fRow, k, d)) + if bool(_fGet("neutralize", False)): + _opType = OTE.NEUTRALIZATION_IMAGE + logger.info(f"describeImage: file {fileId} has neutralize=True, using NEUTRALIZATION_IMAGE (internal models only)") + except Exception: + pass + + visionRequest = AiCallRequest( + prompt=prompt, + options=AiCallOptions(operationType=_opType), + messages=[{"role": "user", "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": dataUrl}}, + ]}], + ) + visionResponse = await services.ai.callAi(visionRequest) + + if visionResponse.errorCount > 0: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error=visionResponse.content) + return ToolResult(toolCallId="", toolName="describeImage", success=True, data=visionResponse.content) + + except Exception as e: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error=str(e)) + + registry.register( + "describeImage", _describeImage, + description="Analyze an image using AI vision. Works with image files and images extracted from PDFs/DOCX/PPTX. Use for OCR, data extraction, and visual analysis.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID containing the image or document with images"}, + "prompt": {"type": "string", "description": "What to look for in the image (default: describe everything)"}, + "pageIndex": {"type": "integer", "description": "Filter images by page index (0-based, for multi-page documents)"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py new file mode 100644 index 00000000..f8ee357d --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py @@ -0,0 +1,160 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Feature Data Sub-Agent tool (queryFeatureInstance).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerFeatureSubAgentTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- Feature Data Sub-Agent tool ---- + + async def _queryFeatureInstance(args: Dict[str, Any], context: Dict[str, Any]): + """Delegate a question to the Feature Data Sub-Agent.""" + featureInstanceId = args.get("featureInstanceId", "") + question = args.get("question", "") + if not featureInstanceId or not question: + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error="featureInstanceId and question are required", + ) + try: + from modules.serviceCenter.services.serviceAgent.featureDataAgent import runFeatureDataAgent + from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource + from modules.interfaces.interfaceDbApp import getRootInterface + + rootIf = getRootInterface() + instance = rootIf.getFeatureInstance(featureInstanceId) + if not instance: + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error=f"Feature instance {featureInstanceId} not found", + ) + + featureCode = instance.featureCode + mandateId = instance.mandateId or "" + instanceLabel = instance.label or "" + userId = context.get("userId", "") + workspaceInstanceId = context.get("featureInstanceId", "") + + rootDbConn = rootIf.db if hasattr(rootIf, "db") else None + if rootDbConn is None: + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error="No database connector available", + ) + + featureDataSources = rootDbConn.getRecordset( + FeatureDataSource, + recordFilter={"featureInstanceId": featureInstanceId, "workspaceInstanceId": workspaceInstanceId}, + ) + + _anySourceNeutralize = any( + bool(ds.get("neutralize", False) if isinstance(ds, dict) else getattr(ds, "neutralize", False)) + for ds in (featureDataSources or []) + ) + + from modules.security.rbacCatalog import getCatalogService + catalog = getCatalogService() + tableFilters = {} + if not featureDataSources: + selectedTables = catalog.getDataObjects(featureCode) + else: + allObjs = {o["meta"]["table"]: o for o in catalog.getDataObjects(featureCode) if "meta" in o and "table" in o.get("meta", {})} + selectedTables = [allObjs[ds["tableName"]] for ds in featureDataSources if ds.get("tableName") in allObjs] + for ds in featureDataSources: + rf = ds.get("recordFilter") + if rf and isinstance(rf, dict) and ds.get("tableName"): + tableFilters[ds["tableName"]] = rf + + if not selectedTables: + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error=f"No data tables available for feature '{featureCode}'", + ) + + from modules.connectors.connectorDbPostgre import DatabaseConnector + from modules.shared.configuration import APP_CONFIG + featureDbName = f"poweron_{featureCode.lower()}" + featureDbConn = DatabaseConnector( + dbHost=APP_CONFIG.get("DB_HOST", "localhost"), + dbDatabase=featureDbName, + dbUser=APP_CONFIG.get("DB_USER"), + dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"), + dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), + userId=userId or "agent", + ) + + aiService = services.ai if hasattr(services, "ai") else None + if aiService is None: + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error="AI service not available for sub-agent", + ) + + async def _subAgentAiCall(req): + if _anySourceNeutralize: + req.requireNeutralization = True + return await aiService.callAi(req) + + try: + answer = await runFeatureDataAgent( + question=question, + featureInstanceId=featureInstanceId, + featureCode=featureCode, + selectedTables=selectedTables, + mandateId=mandateId, + userId=userId, + aiCallFn=_subAgentAiCall, + dbConnector=featureDbConn, + instanceLabel=instanceLabel, + tableFilters=tableFilters, + ) + finally: + try: + featureDbConn.close() + except Exception: + pass + + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=True, data=answer, + ) + except Exception as e: + logger.error(f"queryFeatureInstance failed: {e}", exc_info=True) + return ToolResult( + toolCallId="", toolName="queryFeatureInstance", + success=False, error=str(e), + ) + + registry.register( + "queryFeatureInstance", _queryFeatureInstance, + description=( + "Query data from a feature instance (e.g. Trustee, CommCoach). " + "Delegates to a specialized sub-agent that knows the feature's data schema " + "and can browse/query its tables. Use this when the user has attached " + "feature data sources or asks about feature-specific data." + ), + parameters={ + "type": "object", + "properties": { + "featureInstanceId": {"type": "string", "description": "ID of the feature instance to query"}, + "question": {"type": "string", "description": "What data to find or analyze from this feature instance"}, + }, + "required": ["featureInstanceId", "question"] + }, + readOnly=True + ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_helpers.py b/modules/serviceCenter/services/serviceAgent/coreTools/_helpers.py new file mode 100644 index 00000000..b0c556f5 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_helpers.py @@ -0,0 +1,62 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Shared helpers for core agent tools (file scope, binary detection, temp folder).""" + +import logging +from typing import Optional + +_MAX_TOOL_RESULT_CHARS = 50_000 + +_BINARY_SIGNATURES = (b"%PDF", b"\x89PNG", b"\xff\xd8\xff", b"GIF8", b"PK\x03\x04", b"Rar!", b"\x1f\x8b") + + +def _resolveFileScope(fileId: str, context: dict) -> tuple: + """Resolve featureInstanceId and mandateId for a file from context or management DB. + + Returns (featureInstanceId, mandateId) — never None, always strings. + """ + fiId = context.get("featureInstanceId", "") or "" + mId = context.get("mandateId", "") or "" + if fiId and mId: + return fiId, mId + try: + from modules.datamodels.datamodelFiles import FileItem + from modules.interfaces.interfaceDbManagement import ComponentObjects + fm = ComponentObjects().db._loadRecord(FileItem, fileId) + if fm: + _get = (lambda k: fm.get(k, "")) if isinstance(fm, dict) else (lambda k: getattr(fm, k, "")) + fiId = fiId or str(_get("featureInstanceId") or "") + mId = mId or str(_get("mandateId") or "") + except Exception: + pass + return fiId, mId + + +def _looksLikeBinary(data: bytes, sampleSize: int = 1024) -> bool: + """Detect binary content by checking for magic bytes and non-printable char ratio.""" + if any(data[:8].startswith(sig) for sig in _BINARY_SIGNATURES): + return True + sample = data[:sampleSize] + if not sample: + return False + nonPrintable = sum(1 for b in sample if b < 0x09 or (0x0E <= b < 0x20 and b != 0x1B)) + return nonPrintable / len(sample) > 0.10 + + +def _getOrCreateTempFolder(chatService) -> Optional[str]: + """Return the ID of the root-level 'Temp' folder, creating it if it doesn't exist.""" + try: + allFolders = chatService.interfaceDbComponent.listFolders() + tempFolder = next( + (f for f in allFolders + if f.get("name") == "Temp" and not f.get("parentId")), + None, + ) + if tempFolder: + return tempFolder.get("id") + newFolder = chatService.interfaceDbComponent.createFolder("Temp", parentId=None) + return newFolder.get("id") if newFolder else None + except Exception as e: + logger.warning(f"Could not get/create Temp folder: {e}") + return None + diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py new file mode 100644 index 00000000..20f56d1a --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py @@ -0,0 +1,958 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Media and utility tools (render, TTS, STT, image gen, charts, neutralize, code exec).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +def _registerMediaTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + # ---- Document rendering tool ---- + + def _markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]: + """Convert markdown content to the standard document JSON format expected by renderers.""" + import re as _re + + sections = [] + order = 0 + lines = markdown.split("\n") + i = 0 + + def _nextId(): + nonlocal order + order += 1 + return f"s_{order}" + + while i < len(lines): + line = lines[i] + + # --- Headings --- + headingMatch = _re.match(r'^(#{1,6})\s+(.+)', line) + if headingMatch: + level = len(headingMatch.group(1)) + text = headingMatch.group(2).strip() + sections.append({ + "id": _nextId(), "content_type": "heading", "order": order, + "elements": [{"content": {"text": text, "level": level}}], + }) + i += 1 + continue + + # --- Fenced code blocks --- + codeMatch = _re.match(r'^```(\w*)', line) + if codeMatch: + lang = codeMatch.group(1) or "text" + codeLines = [] + i += 1 + while i < len(lines) and not lines[i].startswith("```"): + codeLines.append(lines[i]) + i += 1 + i += 1 + sections.append({ + "id": _nextId(), "content_type": "code_block", "order": order, + "elements": [{"content": {"code": "\n".join(codeLines), "language": lang}}], + }) + continue + + # --- Tables --- + tableMatch = _re.match(r'^\|(.+)\|$', line) + if tableMatch and (i + 1) < len(lines) and _re.match(r'^\|[\s\-:|]+\|$', lines[i + 1]): + headerCells = [c.strip() for c in tableMatch.group(1).split("|")] + i += 2 + rows = [] + while i < len(lines) and _re.match(r'^\|(.+)\|$', lines[i]): + rowCells = [c.strip() for c in lines[i][1:-1].split("|")] + rows.append(rowCells) + i += 1 + sections.append({ + "id": _nextId(), "content_type": "table", "order": order, + "elements": [{"content": {"headers": headerCells, "rows": rows}}], + }) + continue + + # --- Bullet / numbered lists --- + listMatch = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', line) + if listMatch: + isNumbered = bool(_re.match(r'\d+[.)]', listMatch.group(2))) + items = [] + while i < len(lines) and _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]): + m = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]) + items.append({"text": m.group(3).strip()}) + i += 1 + sections.append({ + "id": _nextId(), "content_type": "bullet_list", "order": order, + "elements": [{"content": {"items": items, "list_type": "numbered" if isNumbered else "bullet"}}], + }) + continue + + # --- Empty lines (skip) --- + if not line.strip(): + i += 1 + continue + + # --- Images: ![alt](file:fileId) or ![alt](url) --- + imgMatch = _re.match(r'^!\[([^\]]*)\]\(([^)]+)\)', line) + if imgMatch: + altText = imgMatch.group(1).strip() or "Image" + src = imgMatch.group(2).strip() + fileId = "" + if src.startswith("file:"): + fileId = src[5:] + sections.append({ + "id": _nextId(), "content_type": "image", "order": order, + "elements": [{ + "content": { + "altText": altText, + "base64Data": "", + "_fileRef": fileId, + "_srcUrl": src if not fileId else "", + } + }], + }) + i += 1 + continue + + # --- Paragraph (collect consecutive non-empty lines) --- + paraLines = [] + while i < len(lines) and lines[i].strip() and not _re.match(r'^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )', lines[i]): + paraLines.append(lines[i]) + i += 1 + if paraLines: + sections.append({ + "id": _nextId(), "content_type": "paragraph", "order": order, + "elements": [{"content": {"text": " ".join(paraLines)}}], + }) + continue + + i += 1 + + if not sections: + sections.append({ + "id": _nextId(), "content_type": "paragraph", "order": order, + "elements": [{"content": {"text": markdown.strip() or "(empty)"}}], + }) + + return { + "metadata": { + "split_strategy": "single_document", + "source_documents": [], + "extraction_method": "agent_rendering", + "title": title, + "language": language, + }, + "documents": [{ + "id": "doc_1", + "title": title, + "sections": sections, + }], + } + + async def _renderDocument(args: Dict[str, Any], context: Dict[str, Any]): + """Render agent-produced markdown content into any document format via the RendererRegistry.""" + import re as _re + sourceFileId = (args.get("sourceFileId") or "").strip() + content = args.get("content", "") + if not isinstance(content, str): + content = str(content) if content is not None else "" + outputFormat = args.get("outputFormat", "pdf") + title = args.get("title", "Document") + language = args.get("language", "de") + + if sourceFileId: + try: + dbMgmt = services.chat.interfaceDbComponent + fileRow = dbMgmt.getFile(sourceFileId) + if not fileRow: + return ToolResult( + toolCallId="", + toolName="renderDocument", + success=False, + error=f"sourceFileId not found: {sourceFileId}", + ) + rawBytes = dbMgmt.getFileData(sourceFileId) + if not rawBytes: + return ToolResult( + toolCallId="", + toolName="renderDocument", + success=False, + error=f"sourceFileId has no data: {sourceFileId}", + ) + try: + content = rawBytes.decode("utf-8") + except UnicodeDecodeError: + content = rawBytes.decode("latin-1", errors="replace") + except Exception as e: + return ToolResult( + toolCallId="", + toolName="renderDocument", + success=False, + error=f"Could not read sourceFileId: {e}", + ) + + if not (content or "").strip(): + return ToolResult( + toolCallId="", + toolName="renderDocument", + success=False, + error=( + "Provide non-empty `content` (markdown) or `sourceFileId` (id of a .md/.txt from writeFile). " + "For long documents use writeFile create+append, then renderDocument(sourceFileId=...)." + ), + ) + + modelMaxTokens = context.get("modelMaxOutputTokens", 0) + _inlineCharLimit = int(modelMaxTokens * 3 * 0.5) if modelMaxTokens > 0 else 6000 + _inlineCharLimit = max(_inlineCharLimit, 3000) + + if not sourceFileId and len(content) > _inlineCharLimit: + return ToolResult( + toolCallId="", + toolName="renderDocument", + success=False, + error=( + f"Inline `content` is {len(content)} chars — over the {_inlineCharLimit} char limit " + f"(derived from model output budget of {modelMaxTokens} tokens). " + "Large documents must use the file path:\n" + "1. writeFile(mode='create', name='draft.md', content=)\n" + "2. writeFile(mode='append', fileId=, content=) — repeat as needed\n" + "3. renderDocument(sourceFileId=, outputFormat='pdf', title='...')\n" + "This avoids output truncation entirely." + ), + ) + + try: + structuredContent = _markdownToDocumentJson(content, title, language) + + # Resolve image file references (file:fileId) to base64 data from Knowledge Store + knowledgeService = None + try: + knowledgeService = services.getService("knowledge") + except Exception: + pass + resolvedImages = 0 + for doc in structuredContent.get("documents", []): + for section in doc.get("sections", []): + if section.get("content_type") != "image": + continue + for element in section.get("elements", []): + contentObj = element.get("content", {}) + fileRef = contentObj.get("_fileRef", "") + if not fileRef or contentObj.get("base64Data"): + continue + if knowledgeService: + chunks = knowledgeService._knowledgeDb.getContentChunks(fileRef) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if imageChunks: + contentObj["base64Data"] = imageChunks[0].get("data", "") + chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType", "image/png") + contentObj["mimeType"] = chunkMime + resolvedImages += 1 + if not contentObj.get("base64Data"): + try: + rawBytes = services.chat.getFileData(fileRef) + if rawBytes: + import base64 as _b64 + contentObj["base64Data"] = _b64.b64encode(rawBytes).decode("ascii") + contentObj["mimeType"] = "image/png" + resolvedImages += 1 + except Exception: + pass + contentObj.pop("_fileRef", None) + contentObj.pop("_srcUrl", None) + + sectionCount = len(structuredContent.get("documents", [{}])[0].get("sections", [])) + logger.info(f"renderDocument: parsed {sectionCount} sections from markdown ({len(content)} chars), resolved {resolvedImages} image(s), format={outputFormat}") + + generationService = services.getService("generation") + documents = await generationService.renderReport( + extractedContent=structuredContent, + outputFormat=outputFormat, + language=language, + title=title, + userPrompt=content, + ) + + if not documents: + return ToolResult(toolCallId="", toolName="renderDocument", success=False, error="Rendering produced no output") + + savedFiles = [] + sideEvents = [] + chatService = services.chat + + sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "document" + + for doc in documents: + docData = doc.documentData if hasattr(doc, "documentData") else b"" + docName = doc.filename if hasattr(doc, "filename") else f"{sanitizedTitle}.{outputFormat}" + docMime = doc.mimeType if hasattr(doc, "mimeType") else "application/octet-stream" + + if not docName.lower().endswith(f".{outputFormat}"): + docName = f"{sanitizedTitle}.{outputFormat}" + + fileItem = None + if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): + fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) + else: + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) + + if fileItem: + fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) + tempFolderId = _getOrCreateTempFolder(chatService) + if tempFolderId: + chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) + savedFiles.append(f"- {docName} (id: {fid})") + sideEvents.append({ + "type": "fileCreated", + "data": { + "fileId": fid, + "fileName": docName, + "mimeType": docMime, + "fileSize": len(docData), + }, + }) + + result = f"Rendered {len(documents)} document(s):\n" + "\n".join(savedFiles) + return ToolResult(toolCallId="", toolName="renderDocument", success=True, data=result, sideEvents=sideEvents) + + except Exception as e: + logger.error(f"renderDocument failed: {e}") + return ToolResult(toolCallId="", toolName="renderDocument", success=False, error=str(e)) + + registry.register( + "renderDocument", _renderDocument, + description=( + "Render markdown into a document file (PDF, DOCX, XLSX, PPTX, CSV, HTML, MD, JSON, TXT). " + "For long documents: write markdown with writeFile (mode=create then append chunks), then call this tool with " + "`sourceFileId` only (tiny JSON — avoids model output truncation). For short docs you may pass `content` inline. " + "Images: ![alt text](file:fileId) in the markdown." + ), + parameters={ + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Full markdown inline. Prefer `sourceFileId` when the document is large (many KB).", + }, + "sourceFileId": { + "type": "string", + "description": "Chat file id of markdown saved via writeFile (create+append). Use this instead of `content` for long PDFs.", + }, + "outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"}, + "title": {"type": "string", "description": "Document title", "default": "Document"}, + "language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"}, + }, + }, + readOnly=False, + ) + + # ── textToSpeech tool ────────────────────────────────────────────── + def _stripMarkdownForTts(text: str) -> str: + """Strip markdown formatting so TTS reads clean speech text.""" + import re as _re + t = text + t = _re.sub(r'\*\*(.+?)\*\*', r'\1', t) + t = _re.sub(r'\*(.+?)\*', r'\1', t) + t = _re.sub(r'__(.+?)__', r'\1', t) + t = _re.sub(r'_(.+?)_', r'\1', t) + t = _re.sub(r'`[^`]+`', lambda m: m.group(0)[1:-1], t) + t = _re.sub(r'^#{1,6}\s*', '', t, flags=_re.MULTILINE) + t = _re.sub(r'^\s*[-*+]\s+', '', t, flags=_re.MULTILINE) + t = _re.sub(r'^\s*\d+\.\s+', '', t, flags=_re.MULTILINE) + t = _re.sub(r'\[(.+?)\]\(.+?\)', r'\1', t) + t = _re.sub(r'!\[.*?\]\(.*?\)', '', t) + t = _re.sub(r'\n{3,}', '\n\n', t) + return t.strip() + + async def _textToSpeech(args: Dict[str, Any], context: Dict[str, Any]): + """Convert text to speech using Google Cloud TTS, deliver audio via SSE.""" + import base64 as _b64 + text = args.get("text", "") + language = args.get("language", "auto") + voiceName = args.get("voiceName") + + if not text: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is required") + + cleanText = _stripMarkdownForTts(text) + if not cleanText: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is empty after stripping markdown") + + try: + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + mandateId = context.get("mandateId", "") + voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) + + _ISO_TO_BCP47 = { + "de": "de-DE", "en": "en-US", "fr": "fr-FR", "it": "it-IT", + "es": "es-ES", "pt": "pt-BR", "nl": "nl-NL", "pl": "pl-PL", + "ru": "ru-RU", "ja": "ja-JP", "zh": "zh-CN", "ko": "ko-KR", + "ar": "ar-XA", "hi": "hi-IN", "tr": "tr-TR", "sv": "sv-SE", + } + + if language == "auto": + try: + snippet = cleanText[:500] + detectResult = await voiceInterface.detectLanguage(snippet) + if detectResult and detectResult.get("success"): + detected = detectResult.get("language", "de") + language = _ISO_TO_BCP47.get(detected, detected) + if "-" not in language: + language = _ISO_TO_BCP47.get(language, f"{language}-{language.upper()}") + logger.info(f"textToSpeech: auto-detected language '{detected}' -> '{language}'") + else: + language = "de-DE" + except Exception as detectErr: + logger.warning(f"textToSpeech: language detection failed: {detectErr}, defaulting to de-DE") + language = "de-DE" + + if not voiceName: + try: + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.interfaces.interfaceDbApp import getRootInterface + userId = context.get("userId", "") + if userId: + rootIf = getRootInterface() + prefRecords = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId} + ) + if prefRecords: + allPrefs = [ + r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else r + for r in prefRecords + ] + _mid = str(mandateId or "").strip() + scopedPref = next((p for p in allPrefs if str(p.get("mandateId") or "").strip() == _mid), None) + globalPref = next((p for p in allPrefs if not str(p.get("mandateId") or "").strip()), None) + + def _resolveVoiceFromMap(prefDict, lang): + vm = (prefDict or {}).get("ttsVoiceMap", {}) or {} + if not isinstance(vm, dict) or not vm: + return None + baseLang = lang.split("-")[0].lower() if isinstance(lang, str) and lang else "" + langNorm = str(lang or "").strip() + if langNorm in vm: + entry = vm[langNorm] + return entry.get("voiceName") if isinstance(entry, dict) else entry + if baseLang and baseLang in vm: + entry = vm[baseLang] + return entry.get("voiceName") if isinstance(entry, dict) else entry + if baseLang: + for mk, mv in vm.items(): + mkn = str(mk).lower() + if mkn == baseLang or mkn.startswith(f"{baseLang}-"): + return mv.get("voiceName") if isinstance(mv, dict) else mv + return None + + voiceName = ( + _resolveVoiceFromMap(scopedPref, language) + or _resolveVoiceFromMap(globalPref, language) + or _resolveVoiceFromMap(allPrefs[0], language) + ) + if not voiceName: + for candidate in [globalPref, scopedPref, allPrefs[0]]: + if candidate and candidate.get("ttsVoice") and candidate.get("ttsLanguage") == language: + voiceName = candidate["ttsVoice"] + break + if voiceName: + logger.info(f"textToSpeech: using configured voice '{voiceName}' for language '{language}'") + except Exception as prefErr: + logger.debug(f"textToSpeech: could not load voice preferences: {prefErr}") + + ttsResult = await voiceInterface.textToSpeech( + text=cleanText, + languageCode=language, + voiceName=voiceName, + ) + + if not ttsResult or not ttsResult.get("success"): + errMsg = ttsResult.get("error", "TTS call failed") if ttsResult else "TTS returned None" + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=errMsg) + + audioContent = ttsResult.get("audioContent", "") + if not audioContent: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="TTS returned no audio") + + if isinstance(audioContent, bytes): + audioB64 = _b64.b64encode(audioContent).decode("ascii") + elif isinstance(audioContent, str): + audioB64 = audioContent + else: + audioB64 = str(audioContent) + + audioFormat = ttsResult.get("audioFormat", "mp3") + charCount = len(cleanText) + usedVoice = voiceName or "default" + logger.info(f"textToSpeech: generated {audioFormat} audio for {charCount} chars, language={language}, voice={usedVoice}") + + return ToolResult( + toolCallId="", toolName="textToSpeech", success=True, + data=f"Audio generated ({charCount} characters, language={language}, voice={usedVoice}). Playing in chat.", + sideEvents=[{ + "type": "voiceResponse", + "data": { + "audio": audioB64, + "format": audioFormat, + "language": language, + "charCount": charCount, + }, + }], + ) + + except ImportError: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, + error="Voice interface not available (missing dependency)") + except Exception as e: + logger.error(f"textToSpeech failed: {e}") + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=str(e)) + + registry.register( + "textToSpeech", _textToSpeech, + description=( + "Convert text to speech audio. The audio is played directly in the chat. " + "Use this when the user asks you to read something aloud, narrate, or speak. " + "Language is auto-detected from the text content. You do NOT need to specify a language." + ), + parameters={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "The text to convert to speech. Can include markdown (will be stripped automatically)."}, + "language": {"type": "string", "description": "BCP-47 language code (e.g. de-DE, en-US) or 'auto' for automatic detection", "default": "auto"}, + "voiceName": {"type": "string", "description": "Optional specific voice name. If omitted, uses the configured voice for the detected language."}, + }, + "required": ["text"], + }, + readOnly=False, + ) + + # ── generateImage tool ───────────────────────────────────────────── + + async def _generateImage(args: Dict[str, Any], context: Dict[str, Any]): + """Generate an image from a text prompt using AI (DALL-E).""" + import re as _re + + prompt = (args.get("prompt") or "").strip() + style = (args.get("style") or "").strip() or None + title = (args.get("title") or "").strip() or "Generated Image" + + if not prompt: + return ToolResult(toolCallId="", toolName="generateImage", success=False, error="prompt is required") + + try: + from modules.serviceCenter.services.serviceGeneration.paths.imagePath import ImageGenerationPath + + imagePath = ImageGenerationPath(services) + aiResponse = await imagePath.generateImages( + userPrompt=prompt, + count=1, + style=style, + format="png", + title=title, + ) + + if not aiResponse.documents: + return ToolResult(toolCallId="", toolName="generateImage", success=False, error="Image generation returned no image data") + + sideEvents = [] + savedFiles = [] + chatService = services.chat + sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "generated_image" + + for doc in aiResponse.documents: + docData = doc.documentData if hasattr(doc, "documentData") else b"" + docName = doc.documentName if hasattr(doc, "documentName") else f"{sanitizedTitle}.png" + docMime = doc.mimeType if hasattr(doc, "mimeType") else "image/png" + + if not docName.lower().endswith(".png"): + docName = f"{sanitizedTitle}.png" + + fileItem = None + if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): + fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) + else: + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) + + if fileItem: + fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) + tempFolderId = _getOrCreateTempFolder(chatService) + if tempFolderId: + chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) + savedFiles.append(f"- {docName} (id: {fid})") + sideEvents.append({ + "type": "fileCreated", + "data": { + "fileId": fid, + "fileName": docName, + "mimeType": docMime, + "fileSize": len(docData), + }, + }) + + result = f"Generated {len(aiResponse.documents)} image(s):\n" + "\n".join(savedFiles) + return ToolResult(toolCallId="", toolName="generateImage", success=True, data=result, sideEvents=sideEvents) + + except Exception as e: + logger.error(f"generateImage failed: {e}") + return ToolResult(toolCallId="", toolName="generateImage", success=False, error=str(e)) + + registry.register( + "generateImage", _generateImage, + description=( + "Generate an image from a text description using AI (DALL-E). " + "The generated image is saved as a file in the workspace. " + "Use this when the user asks to create, generate, draw, or design an image, illustration, icon, logo, diagram, or any visual content. " + "Provide a detailed, descriptive prompt for best results." + ), + parameters={ + "type": "object", + "properties": { + "prompt": {"type": "string", "description": "Detailed description of the image to generate. Be specific about subject, composition, colors, style, and mood."}, + "style": {"type": "string", "description": "Optional style modifier (e.g. 'photorealistic', 'watercolor', 'digital art', 'minimalist', 'sketch')"}, + "title": {"type": "string", "description": "Title/filename for the generated image", "default": "Generated Image"}, + }, + "required": ["prompt"], + }, + readOnly=False, + ) + + # ── createChart tool ───────────────────────────────────────────────── + + async def _createChart(args: Dict[str, Any], context: Dict[str, Any]): + """Create a data chart as PNG image using matplotlib.""" + import re as _re + + chartType = (args.get("chartType") or "bar").strip().lower() + title = (args.get("title") or "Chart").strip() + labels = args.get("labels") or [] + datasets = args.get("datasets") or [] + xLabel = (args.get("xLabel") or "").strip() + yLabel = (args.get("yLabel") or "").strip() + width = min(max(args.get("width") or 10, 4), 20) + height = min(max(args.get("height") or 6, 3), 14) + colors = args.get("colors") or None + + if not datasets: + return ToolResult(toolCallId="", toolName="createChart", success=False, error="datasets is required (list of {label, values})") + + try: + import matplotlib + matplotlib.use("Agg") + import logging as _mpllog + _mpllog.getLogger("matplotlib").setLevel(_mpllog.WARNING) + import matplotlib.pyplot as plt + import io + + _DEFAULT_COLORS = [ + "#4285F4", "#EA4335", "#FBBC04", "#34A853", "#FF6D01", + "#46BDC6", "#7B61FF", "#F538A0", "#00ACC1", "#AB47BC", + ] + usedColors = colors if colors and len(colors) >= len(datasets) else _DEFAULT_COLORS + + fig, ax = plt.subplots(figsize=(width, height)) + fig.patch.set_facecolor("#FFFFFF") + ax.set_facecolor("#FAFAFA") + + if chartType in ("pie", "donut"): + values = datasets[0].get("values", []) if datasets else [] + explode = [0.02] * len(values) + wedges, texts, autotexts = ax.pie( + values, labels=labels, autopct="%1.1f%%", + colors=usedColors[:len(values)], explode=explode, + textprops={"fontsize": 9}, + ) + if chartType == "donut": + ax.add_artist(plt.Circle((0, 0), 0.55, fc="white")) + ax.set_title(title, fontsize=14, fontweight="bold", pad=16) + + else: + import numpy as _np + x = _np.arange(len(labels)) if labels else _np.arange(max(len(d.get("values", [])) for d in datasets)) + barWidth = 0.8 / max(len(datasets), 1) + + for i, ds in enumerate(datasets): + dsLabel = ds.get("label", f"Series {i+1}") + values = ds.get("values", []) + color = usedColors[i % len(usedColors)] + + if chartType == "bar": + offset = (i - len(datasets) / 2 + 0.5) * barWidth + ax.bar(x + offset, values, barWidth, label=dsLabel, color=color, edgecolor="white", linewidth=0.5) + elif chartType == "horizontalbar": + offset = (i - len(datasets) / 2 + 0.5) * barWidth + ax.barh(x + offset, values, barWidth, label=dsLabel, color=color, edgecolor="white", linewidth=0.5) + elif chartType == "line": + ax.plot(x[:len(values)], values, marker="o", markersize=5, label=dsLabel, color=color, linewidth=2) + elif chartType == "area": + ax.fill_between(x[:len(values)], values, alpha=0.3, color=color) + ax.plot(x[:len(values)], values, label=dsLabel, color=color, linewidth=2) + elif chartType == "scatter": + ax.scatter(x[:len(values)], values, label=dsLabel, color=color, s=50, edgecolors="white", linewidth=0.5) + else: + ax.bar(x, values, label=dsLabel, color=color) + + if labels: + if chartType == "horizontalbar": + ax.set_yticks(x) + ax.set_yticklabels(labels, fontsize=9) + else: + ax.set_xticks(x) + ax.set_xticklabels(labels, fontsize=9, rotation=45 if len(labels) > 6 else 0, ha="right" if len(labels) > 6 else "center") + + ax.set_title(title, fontsize=14, fontweight="bold", pad=12) + if xLabel: + ax.set_xlabel(xLabel, fontsize=10) + if yLabel: + ax.set_ylabel(yLabel, fontsize=10) + if len(datasets) > 1: + ax.legend(fontsize=9, framealpha=0.9) + ax.grid(axis="y", alpha=0.3, linestyle="--") + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + + plt.tight_layout() + buf = io.BytesIO() + fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") + plt.close(fig) + pngData = buf.getvalue() + + chatService = services.chat + sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "chart" + fileName = f"{sanitizedTitle}.png" + + if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): + fileItem = chatService.interfaceDbComponent.saveGeneratedFile(pngData, fileName, "image/png") + else: + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(pngData, fileName) + + fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") if isinstance(fileItem, dict) else "?" + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId and fid != "?": + chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) + tempFolderId = _getOrCreateTempFolder(chatService) + if tempFolderId and fid != "?": + chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) + + sideEvents = [{"type": "fileCreated", "data": { + "fileId": fid, "fileName": fileName, + "mimeType": "image/png", "fileSize": len(pngData), + }}] + return ToolResult( + toolCallId="", toolName="createChart", success=True, + data=f"Chart saved as '{fileName}' (id: {fid}, {len(pngData)} bytes). " + f"Embed in documents with: ![{title}](file:{fid})", + sideEvents=sideEvents, + ) + + except Exception as e: + logger.error(f"createChart failed: {e}", exc_info=True) + return ToolResult(toolCallId="", toolName="createChart", success=False, error=str(e)) + + registry.register( + "createChart", _createChart, + description=( + "Create a data chart/graph as a PNG image using matplotlib. " + "Supported types: bar, horizontalBar, line, area, scatter, pie, donut. " + "The chart is saved as a file in the workspace. " + "Use the returned fileId to embed in documents via renderDocument: ![title](file:fileId). " + "Provide structured data with labels and datasets." + ), + parameters={ + "type": "object", + "properties": { + "chartType": { + "type": "string", + "enum": ["bar", "horizontalBar", "line", "area", "scatter", "pie", "donut"], + "description": "Chart type (default: bar)", + }, + "title": {"type": "string", "description": "Chart title"}, + "labels": { + "type": "array", "items": {"type": "string"}, + "description": "X-axis labels / category names", + }, + "datasets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": {"type": "string", "description": "Series name (legend)"}, + "values": {"type": "array", "items": {"type": "number"}, "description": "Data values"}, + }, + "required": ["values"], + }, + "description": "Data series to plot", + }, + "xLabel": {"type": "string", "description": "X-axis label"}, + "yLabel": {"type": "string", "description": "Y-axis label"}, + "colors": { + "type": "array", "items": {"type": "string"}, + "description": "Custom hex colors for series (e.g. ['#4285F4', '#EA4335'])", + }, + "width": {"type": "number", "description": "Figure width in inches (4-20, default 10)"}, + "height": {"type": "number", "description": "Figure height in inches (3-14, default 6)"}, + }, + "required": ["datasets"], + }, + readOnly=False, + ) + + # ── Phase 3: speechToText, detectLanguage, neutralizeData, executeCode ── + + async def _speechToText(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="speechToText", success=False, error="fileId is required") + try: + chatService = services.chat + audioData = chatService.interfaceDbComponent.getFileData(fileId) + if not audioData: + return ToolResult(toolCallId="", toolName="speechToText", success=False, error=f"No data found for file {fileId}") + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + mandateId = context.get("mandateId", "") + voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) + language = args.get("language", "de-DE") + result = await voiceInterface.speechToText(audioData, language=language) + if result and result.get("success"): + transcript = result.get("text", "") + confidence = result.get("confidence", 0) + return ToolResult( + toolCallId="", toolName="speechToText", success=True, + data=f"Transcript (confidence: {confidence:.0%}):\n{transcript}" + ) + return ToolResult(toolCallId="", toolName="speechToText", success=False, error=result.get("error", "Transcription failed")) + except Exception as e: + return ToolResult(toolCallId="", toolName="speechToText", success=False, error=str(e)) + + async def _detectLanguage(args: Dict[str, Any], context: Dict[str, Any]): + text = args.get("text", "") + if not text: + return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error="text is required") + try: + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + mandateId = context.get("mandateId", "") + voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) + result = await voiceInterface.detectLanguage(text) + if result and result.get("success"): + lang = result.get("language", "unknown") + return ToolResult(toolCallId="", toolName="detectLanguage", success=True, data=f"Detected language: {lang}") + return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error=result.get("error", "Detection failed")) + except Exception as e: + return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error=str(e)) + + async def _neutralizeData(args: Dict[str, Any], context: Dict[str, Any]): + text = args.get("text", "") + fileId = args.get("fileId", "") + if not text and not fileId: + return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="text or fileId is required") + try: + neutralizationService = services.getService("neutralization") + if not neutralizationService: + return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="Neutralization service not available") + if not neutralizationService.interfaceDbComponent: + neutralizationService.interfaceDbComponent = services.chat.interfaceDbComponent + if text: + result = await neutralizationService.processTextAsync(text, fileId or None) + else: + result = neutralizationService.processFile(fileId) + if result: + neutralized = result.get("neutralized_text", "") or result.get("result", str(result)) + return ToolResult(toolCallId="", toolName="neutralizeData", success=True, data=neutralized) + return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="Neutralization returned no result") + except Exception as e: + return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error=str(e)) + + async def _executeCode(args: Dict[str, Any], context: Dict[str, Any]): + code = args.get("code", "") + language = args.get("language", "python") + if not code: + return ToolResult(toolCallId="", toolName="executeCode", success=False, error="code is required") + if language != "python": + return ToolResult(toolCallId="", toolName="executeCode", success=False, error=f"Language '{language}' not supported. Only 'python' is available.") + try: + from modules.serviceCenter.services.serviceAgent.sandboxExecutor import executePython + result = await executePython(code) + if result.get("success"): + output = result.get("output", "(no output)") + return ToolResult(toolCallId="", toolName="executeCode", success=True, data=output) + error = result.get("error", "Execution failed") + tb = result.get("traceback", "") + return ToolResult(toolCallId="", toolName="executeCode", success=False, error=f"{error}\n{tb}" if tb else error) + except Exception as e: + return ToolResult(toolCallId="", toolName="executeCode", success=False, error=str(e)) + + registry.register( + "speechToText", _speechToText, + description="Transcribe an audio file to text using speech recognition. Returns the transcript with confidence score.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "Audio file ID from the workspace"}, + "language": {"type": "string", "description": "BCP-47 language code (e.g. 'de-DE', 'en-US'). Default: 'de-DE'"}, + }, + "required": ["fileId"] + }, + readOnly=True + ) + + registry.register( + "detectLanguage", _detectLanguage, + description="Detect the language of a text snippet. Returns ISO 639-1 code (e.g. 'de', 'en').", + parameters={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "Text to analyze"}, + }, + "required": ["text"] + }, + readOnly=True + ) + + registry.register( + "neutralizeData", _neutralizeData, + description="Anonymize text or file content by replacing personal data (names, addresses, etc.) with placeholders. Non-destructive -- returns the anonymized copy.", + parameters={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "Text to anonymize"}, + "fileId": {"type": "string", "description": "File ID to anonymize (alternative to text)"}, + }, + }, + readOnly=True + ) + + registry.register( + "executeCode", _executeCode, + description=( + "Execute Python code in a sandboxed environment for calculations and data analysis. " + "Available modules: math, statistics, json, csv, re, datetime, collections, itertools, functools, decimal, fractions, random. " + "No file system, network, or OS access. Max 30s execution time. " + "Use print() to produce output." + ), + parameters={ + "type": "object", + "properties": { + "code": {"type": "string", "description": "Python code to execute"}, + "language": {"type": "string", "description": "Programming language (only 'python' supported)", "default": "python"}, + }, + "required": ["code"] + }, + readOnly=True + ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_workspaceTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_workspaceTools.py new file mode 100644 index 00000000..b5a8f49f --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_workspaceTools.py @@ -0,0 +1,950 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Workspace and file management tools (read, write, search, folders, web, translate).""" + +import logging +from typing import Any, Dict, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( + _getOrCreateTempFolder, + _looksLikeBinary, + _resolveFileScope, + _MAX_TOOL_RESULT_CHARS, +) + +logger = logging.getLogger(__name__) + + +import uuid as _uuid + +def _registerWorkspaceTools(registry: ToolRegistry, services): + """Auto-extracted from registerCoreTools.""" + import uuid as _uuid + + # ---- Read-only tools ---- + + def _applyOffsetLimit(text: str, offset: int = None, limit: int = None) -> str: + """Apply line-based offset/limit to text content, returning numbered lines.""" + if offset is None and limit is None: + return None + lines = text.split("\n") + totalLines = len(lines) + startLine = max(0, (offset or 1) - 1) + endLine = min(totalLines, startLine + (limit or 200)) + selected = lines[startLine:endLine] + numbered = "\n".join(f"{i + startLine + 1}|{line}" for i, line in enumerate(selected)) + header = f"[Lines {startLine + 1}-{endLine} of {totalLines} total]\n" + return header + numbered + + async def _readFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + offset = args.get("offset") + limit = args.get("limit") + if not fileId: + return ToolResult(toolCallId="", toolName="readFile", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None + + # 1) Knowledge Store: return already-extracted text chunks + if knowledgeService: + fileStatus = knowledgeService.getFileStatus(fileId) + if fileStatus == "indexed": + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + textChunks = [ + c for c in (chunks or []) + if c.get("contentType") != "image" and c.get("data") + ] + if textChunks: + assembled = "\n\n".join(c["data"] for c in textChunks) + chunked = _applyOffsetLimit(assembled, offset, limit) + if chunked is not None: + return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) + if len(assembled) > _MAX_TOOL_RESULT_CHARS: + assembled = assembled[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(assembled)}. Use offset/limit to read specific sections.]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=assembled, + ) + elif fileStatus in ("processing", "embedding", "extracted"): + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[File {fileId} is currently being processed (status: {fileStatus}). Try again shortly.]", + ) + + # 2) Not indexed yet: try on-demand extraction + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) + if not fileInfo: + return ToolResult(toolCallId="", toolName="readFile", success=True, data="File not found.") + + fileName = fileInfo.get("fileName", fileId) + mimeType = fileInfo.get("mimeType", "") + + _BINARY_TYPES = ("application/pdf", "image/", "application/vnd.", "application/zip", + "application/x-zip", "application/x-tar", "application/x-7z", + "application/msword", "application/octet-stream", + "message/rfc822") + isBinary = any(mimeType.startswith(t) for t in _BINARY_TYPES) + + rawBytes = chatService.getFileData(fileId) + if not rawBytes: + return ToolResult(toolCallId="", toolName="readFile", success=True, data="File data not accessible.") + + if not isBinary: + isBinary = _looksLikeBinary(rawBytes) + + if isBinary: + try: + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry, ChunkerRegistry + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction + from modules.datamodels.datamodelExtraction import ExtractionOptions + + extracted = runExtraction( + ExtractorRegistry(), ChunkerRegistry(), + rawBytes, fileName, mimeType, ExtractionOptions(), + ) + + contentObjects = [] + for part in extracted.parts: + tg = (part.typeGroup or "").lower() + ct = "image" if tg == "image" else "text" + if not part.data or not part.data.strip(): + continue + contentObjects.append({ + "contentObjectId": part.id, + "contentType": ct, + "data": part.data, + "contextRef": { + "containerPath": fileName, + "location": part.label or "file", + **(part.metadata or {}), + }, + }) + + if contentObjects: + if knowledgeService: + try: + userId = context.get("userId", "") + _fiId, _mId = _resolveFileScope(fileId, context) + await knowledgeService.indexFile( + fileId=fileId, fileName=fileName, mimeType=mimeType, + userId=userId, contentObjects=contentObjects, + featureInstanceId=_fiId, + mandateId=_mId, + ) + except Exception: + pass + + joined = "" + if knowledgeService: + _chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + _textChunks = [ + c for c in (_chunks or []) + if c.get("contentType") != "image" and c.get("data") + ] + if _textChunks: + joined = "\n\n".join(c["data"] for c in _textChunks) + if not joined: + textParts = [o["data"] for o in contentObjects if o["contentType"] != "image"] + joined = "\n\n".join(textParts) if textParts else "" + if joined: + chunked = _applyOffsetLimit(joined, offset, limit) + if chunked is not None: + return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) + if len(joined) > _MAX_TOOL_RESULT_CHARS: + joined = joined[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(joined)}. Use offset/limit to read specific sections.]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=joined, + ) + imgCount = sum(1 for o in contentObjects if o["contentType"] == "image") + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[Extracted {len(contentObjects)} content objects from '{fileName}' " + f"({imgCount} images, no readable text). " + f"Use describeImage(fileId='{fileId}') to analyze visual content.]", + ) + except Exception as extractErr: + logger.warning(f"readFile extraction failed for {fileId} ({fileName}): {extractErr}") + + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[Binary file: '{fileName}', type={mimeType}, size={len(rawBytes)} bytes. " + f"Text extraction not available. Use describeImage for images.]", + ) + + # 3) Text file: decode raw bytes + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + text = rawBytes.decode(encoding) + if text.strip(): + _fileNeedNeutralize = False + try: + from modules.datamodels.datamodelFiles import FileItem as _FI + from modules.interfaces.interfaceDbManagement import ComponentObjects as _CO + _fRec = _CO().db._loadRecord(_FI, fileId) + if _fRec: + _fG = (lambda k, d=None: _fRec.get(k, d)) if isinstance(_fRec, dict) else (lambda k, d=None: getattr(_fRec, k, d)) + _fileNeedNeutralize = bool(_fG("neutralize", False)) + except Exception: + pass + if _fileNeedNeutralize: + try: + _nSvc = services.getService("neutralization") if hasattr(services, "getService") else None + if _nSvc and hasattr(_nSvc, 'processTextAsync'): + _nResult = await _nSvc.processTextAsync(text, fileId) + if _nResult and _nResult.get("neutralized_text"): + text = _nResult["neutralized_text"] + logger.debug(f"readFile: neutralized text for file {fileId}") + else: + logger.warning(f"readFile: neutralization failed for file {fileId}, blocking text (fail-safe)") + return ToolResult(toolCallId="", toolName="readFile", success=True, + data="[File requires neutralization but neutralization failed. Content blocked for data protection.]") + else: + logger.warning(f"readFile: neutralization required but service unavailable for file {fileId}") + return ToolResult(toolCallId="", toolName="readFile", success=True, + data="[File requires neutralization but service unavailable. Content blocked for data protection.]") + except Exception as _nErr: + logger.error(f"readFile: neutralization error for file {fileId}: {_nErr}") + return ToolResult(toolCallId="", toolName="readFile", success=True, + data="[File requires neutralization but an error occurred. Content blocked for data protection.]") + chunked = _applyOffsetLimit(text, offset, limit) + if chunked is not None: + return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) + if len(text) > _MAX_TOOL_RESULT_CHARS: + text = text[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(text)}. Use offset/limit to read specific sections.]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=text, + ) + except (UnicodeDecodeError, ValueError): + continue + + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data="File is empty or could not be decoded.", + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="readFile", success=False, error=str(e)) + + async def _listFiles(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + files = chatService.listFiles( + folderId=args.get("folderId"), + tags=args.get("tags"), + search=args.get("search"), + ) + fileList = "\n".join( + f"- {f.get('fileName', 'unknown')} (id: {f.get('id', '?')}, " + f"type: {f.get('mimeType', '?')}, size: {f.get('fileSize', '?')}, " + f"tags: {f.get('tags', [])}, status: {f.get('status', 'n/a')})" + for f in files + ) if files else "No files found." + return ToolResult(toolCallId="", toolName="listFiles", success=True, data=fileList) + except Exception as e: + return ToolResult(toolCallId="", toolName="listFiles", success=False, error=str(e)) + + async def _searchInFileContent(args: Dict[str, Any], context: Dict[str, Any]): + import re as _re + fileId = args.get("fileId", "") + query = args.get("query", "") + contextLines = args.get("contextLines", 2) + if not fileId or not query: + return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error="fileId and query are required") + try: + chatService = services.chat + rawBytes = chatService.getFileData(fileId) + if not rawBytes: + return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error="File data not accessible") + try: + content = rawBytes.decode("utf-8") + except UnicodeDecodeError: + content = rawBytes.decode("latin-1", errors="replace") + + lines = content.split("\n") + pattern = _re.compile(_re.escape(query), _re.IGNORECASE) + matches = [] + for i, line in enumerate(lines): + if pattern.search(line): + start = max(0, i - contextLines) + end = min(len(lines), i + contextLines + 1) + snippet = "\n".join(f"{j + 1}|{lines[j]}" for j in range(start, end)) + matches.append(snippet) + + if not matches: + return ToolResult(toolCallId="", toolName="searchInFileContent", success=True, + data=f"No matches for '{query}' in file.") + + shown = matches[:20] + resultText = f"Found {len(matches)} match(es) for '{query}':\n\n" + "\n---\n".join(shown) + if len(matches) > 20: + resultText += f"\n\n... and {len(matches) - 20} more matches" + return ToolResult(toolCallId="", toolName="searchInFileContent", success=True, data=resultText) + except Exception as e: + return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error=str(e)) + + async def _listFolders(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + folders = chatService.listFolders(parentId=args.get("parentId")) + folderList = "\n".join( + f"- {f.get('name', 'unnamed')} (id: {f.get('id', '?')})" + for f in folders + ) if folders else "No folders found." + return ToolResult(toolCallId="", toolName="listFolders", success=True, data=folderList) + except Exception as e: + return ToolResult(toolCallId="", toolName="listFolders", success=False, error=str(e)) + + async def _webSearch(args: Dict[str, Any], context: Dict[str, Any]): + query = args.get("query", "") + if not query: + return ToolResult(toolCallId="", toolName="webSearch", success=False, error="query is required") + try: + webService = services.getService("web") + result = await webService.performWebResearch( + prompt=query, + urls=[], + country=None, + language=args.get("language"), + ) + summary = result.get("summary", "") if isinstance(result, dict) else str(result) + return ToolResult( + toolCallId="", toolName="webSearch", success=True, + data=summary or str(result) + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="webSearch", success=False, error=str(e)) + + # ---- Write tools ---- + + async def _tagFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + tags = args.get("tags", []) + if not fileId: + return ToolResult(toolCallId="", toolName="tagFile", success=False, error="fileId is required") + try: + chatService = services.chat + chatService.interfaceDbComponent.updateFile(fileId, {"tags": tags}) + return ToolResult( + toolCallId="", toolName="tagFile", success=True, + data=f"Tags updated to {tags} for file {fileId}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="tagFile", success=False, error=str(e)) + + async def _moveFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + targetFolderId = args.get("targetFolderId") + if not fileId: + return ToolResult(toolCallId="", toolName="moveFile", success=False, error="fileId is required") + try: + chatService = services.chat + chatService.interfaceDbComponent.updateFile(fileId, {"folderId": targetFolderId}) + return ToolResult( + toolCallId="", toolName="moveFile", success=True, + data=f"File {fileId} moved to folder {targetFolderId or 'root'}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="moveFile", success=False, error=str(e)) + + async def _createFolder(args: Dict[str, Any], context: Dict[str, Any]): + name = args.get("name", "") + if not name: + return ToolResult(toolCallId="", toolName="createFolder", success=False, error="name is required") + try: + chatService = services.chat + folder = chatService.createFolder(name=name, parentId=args.get("parentId")) + return ToolResult( + toolCallId="", toolName="createFolder", success=True, + data=f"Folder '{name}' created (id: {folder.get('id', '?')})" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="createFolder", success=False, error=str(e)) + + async def _writeFile(args: Dict[str, Any], context: Dict[str, Any]): + content = args.get("content", "") + mode = args.get("mode", "create") + fileId = args.get("fileId", "") + name = args.get("name", "") + + if not content: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error="content is required") + + try: + chatService = services.chat + dbMgmt = chatService.interfaceDbComponent + + if mode == "append": + if not fileId: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error="fileId is required for mode=append") + file = dbMgmt.getFile(fileId) + if not file: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error=f"File {fileId} not found") + existingData = dbMgmt.getFileData(fileId) or b"" + try: + existingText = existingData.decode("utf-8") + except UnicodeDecodeError: + existingText = existingData.decode("latin-1", errors="replace") + newContent = existingText + content + dbMgmt.updateFileData(fileId, newContent.encode("utf-8")) + dbMgmt.updateFile(fileId, {"fileSize": len(newContent.encode("utf-8"))}) + return ToolResult( + toolCallId="", toolName="writeFile", success=True, + data=f"Appended {len(content)} chars to '{file.fileName}' (id: {fileId}, total: {len(newContent)} chars)", + sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": file.fileName}}], + ) + + if mode == "overwrite": + if not fileId: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error="fileId is required for mode=overwrite") + file = dbMgmt.getFile(fileId) + if not file: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error=f"File {fileId} not found") + dbMgmt.updateFileData(fileId, content.encode("utf-8")) + dbMgmt.updateFile(fileId, {"fileSize": len(content.encode("utf-8"))}) + return ToolResult( + toolCallId="", toolName="writeFile", success=True, + data=f"Overwritten '{file.fileName}' (id: {fileId}, {len(content)} chars)", + sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": file.fileName}}], + ) + + # mode == "create" (default) + if not name: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error="name is required for mode=create") + fileItem, _ = dbMgmt.saveUploadedFile(content.encode("utf-8"), name) + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + dbMgmt.updateFile(fileItem.id, {"featureInstanceId": fiId}) + if args.get("folderId"): + dbMgmt.updateFile(fileItem.id, {"folderId": args["folderId"]}) + if args.get("tags"): + dbMgmt.updateFile(fileItem.id, {"tags": args["tags"]}) + return ToolResult( + toolCallId="", toolName="writeFile", success=True, + data=f"File '{name}' created (id: {fileItem.id})", + sideEvents=[{ + "type": "fileCreated", + "data": { + "fileId": fileItem.id, + "fileName": name, + "mimeType": fileItem.mimeType, + "fileSize": fileItem.fileSize, + }, + }], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error=str(e)) + + # ---- Register all tools ---- + + registry.register( + "readFile", _readFile, + description=( + "Read the content of a file. Returns full content by default. " + "For large files, use offset and limit to read specific line ranges. " + "When truncated, the response tells the total line count so you can paginate." + ), + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to read"}, + "offset": {"type": "integer", "description": "Start reading from this line number (1-based). Omit for full file."}, + "limit": {"type": "integer", "description": "Max number of lines to return (default: all). Use with offset for chunked reading."}, + }, + "required": ["fileId"] + }, + readOnly=True + ) + + registry.register( + "listFiles", _listFiles, + description=( + "List files in the local workspace. Filter by folder, tags, or search term. " + "For external data sources, use browseDataSource instead." + ), + parameters={ + "type": "object", + "properties": { + "folderId": {"type": "string", "description": "Filter by folder ID"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Filter by tags (any match)"}, + "search": {"type": "string", "description": "Search in file names and descriptions"}, + } + }, + readOnly=True + ) + + registry.register( + "searchInFileContent", _searchInFileContent, + description=( + "Search for text within a file's content. Returns matching lines with context. " + "Case-insensitive. Use to locate specific text before using replaceInFile, " + "or to find relevant sections in a large file before reading with offset/limit." + ), + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to search in"}, + "query": {"type": "string", "description": "Text to search for (case-insensitive)"}, + "contextLines": {"type": "integer", "description": "Number of context lines around each match (default: 2)"}, + }, + "required": ["fileId", "query"] + }, + readOnly=True + ) + + registry.register( + "listFolders", _listFolders, + description="List folders in the local workspace. For external data sources, use browseDataSource instead.", + parameters={ + "type": "object", + "properties": { + "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, + } + }, + readOnly=True + ) + + registry.register( + "webSearch", _webSearch, + description="Search the web for general information. Use readUrl to fetch content from a known URL instead.", + parameters={ + "type": "object", + "properties": {"query": {"type": "string", "description": "Search query"}}, + "required": ["query"] + }, + readOnly=True + ) + + registry.register( + "tagFile", _tagFile, + description="Set or update tags on a file for categorization and filtering via listFiles.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags to set"}, + }, + "required": ["fileId", "tags"] + }, + readOnly=False + ) + + registry.register( + "moveFile", _moveFile, + description="Move a file to a different folder in the local workspace.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to move"}, + "targetFolderId": {"type": "string", "description": "Target folder ID (null for root)"}, + }, + "required": ["fileId"] + }, + readOnly=False + ) + + registry.register( + "createFolder", _createFolder, + description="Create a new folder in the local workspace.", + parameters={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "Folder name"}, + "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, + }, + "required": ["name"] + }, + readOnly=False + ) + + registry.register( + "writeFile", _writeFile, + description=( + "Create, append, or overwrite a file. Modes:\n" + "- create (default): create a new file (name required).\n" + "- append: append content to an existing file (fileId required). " + "Use for large content that exceeds a single tool call (~8000 chars per call).\n" + "- overwrite: replace entire file content (fileId required)." + ), + parameters={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "File name (required for mode=create)"}, + "content": {"type": "string", "description": "Content to write/append"}, + "mode": {"type": "string", "enum": ["create", "append", "overwrite"], "description": "Write mode (default: create)"}, + "fileId": {"type": "string", "description": "File ID (required for mode=append/overwrite)"}, + "folderId": {"type": "string", "description": "Target folder ID (mode=create only)"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags (mode=create only)"}, + }, + "required": ["content"] + }, + readOnly=False + ) + + # ---- Phase 1: deleteFile, renameFile, readUrl, translateText ---- + + async def _deleteFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="deleteFile", success=False, error="fileId is required") + try: + chatService = services.chat + file = chatService.interfaceDbComponent.getFile(fileId) + if not file: + return ToolResult(toolCallId="", toolName="deleteFile", success=False, error=f"File {fileId} not found") + fileName = file.fileName + try: + knowledgeService = services.getService("knowledge") + if knowledgeService and hasattr(knowledgeService, "removeFile"): + knowledgeService.removeFile(fileId) + except Exception: + pass + chatService.interfaceDbComponent.deleteFile(fileId) + return ToolResult( + toolCallId="", toolName="deleteFile", success=True, + data=f"File '{fileName}' (id: {fileId}) deleted", + sideEvents=[{"type": "fileDeleted", "data": {"fileId": fileId, "fileName": fileName}}], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="deleteFile", success=False, error=str(e)) + + async def _renameFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + newName = args.get("newName", "") + if not fileId or not newName: + return ToolResult(toolCallId="", toolName="renameFile", success=False, error="fileId and newName are required") + try: + chatService = services.chat + chatService.interfaceDbComponent.updateFile(fileId, {"fileName": newName}) + return ToolResult( + toolCallId="", toolName="renameFile", success=True, + data=f"File {fileId} renamed to '{newName}'", + sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": newName}}], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="renameFile", success=False, error=str(e)) + + async def _readUrl(args: Dict[str, Any], context: Dict[str, Any]): + url = args.get("url", "") + if not url: + return ToolResult(toolCallId="", toolName="readUrl", success=False, error="url is required") + try: + webService = services.getService("web") + result = await webService._performWebCrawl( + instruction="Extract all content from this page", + urls=[url], + maxDepth=1, + maxWidth=1, + ) + if isinstance(result, list) and result: + content = "\n\n".join( + item.get("content", "") or item.get("text", "") or str(item) + for item in result if item + ) + elif isinstance(result, dict): + content = result.get("content", "") or result.get("summary", "") or str(result) + else: + content = str(result) if result else "No content retrieved" + _MAX = 30000 + if len(content) > _MAX: + content = content[:_MAX] + f"\n\n... (truncated at {_MAX} chars)" + return ToolResult(toolCallId="", toolName="readUrl", success=True, data=content) + except Exception as e: + return ToolResult(toolCallId="", toolName="readUrl", success=False, error=str(e)) + + async def _translateText(args: Dict[str, Any], context: Dict[str, Any]): + text = args.get("text", "") + targetLanguage = args.get("targetLanguage", "") + if not text or not targetLanguage: + return ToolResult(toolCallId="", toolName="translateText", success=False, error="text and targetLanguage are required") + try: + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + mandateId = context.get("mandateId", "") + voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) + sourceLanguage = args.get("sourceLanguage", "auto") + result = await voiceInterface.translateText(text, sourceLanguage=sourceLanguage, targetLanguage=targetLanguage) + if result and result.get("success"): + translated = result.get("translated_text", "") + return ToolResult(toolCallId="", toolName="translateText", success=True, data=translated) + return ToolResult(toolCallId="", toolName="translateText", success=False, error=result.get("error", "Translation failed")) + except Exception as e: + return ToolResult(toolCallId="", toolName="translateText", success=False, error=str(e)) + + registry.register( + "deleteFile", _deleteFile, + description="Permanently delete a file from the local workspace.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to delete"}, + }, + "required": ["fileId"] + }, + readOnly=False + ) + + registry.register( + "renameFile", _renameFile, + description="Rename a file in the local workspace. Include the file extension in the new name.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to rename"}, + "newName": {"type": "string", "description": "New file name including extension"}, + }, + "required": ["fileId", "newName"] + }, + readOnly=False + ) + + registry.register( + "readUrl", _readUrl, + description=( + "Read and extract content from a specific URL. " + "Use when the user provides a specific URL to read, or when you need to fetch content from a known web page. " + "For general information searches, use webSearch instead." + ), + parameters={ + "type": "object", + "properties": { + "url": {"type": "string", "description": "The URL to read"}, + }, + "required": ["url"] + }, + readOnly=True + ) + + registry.register( + "translateText", _translateText, + description=( + "Translate text to a target language using Google Cloud Translation. " + "More efficient than AI translation for large text volumes. " + "Use ISO language codes (e.g. 'en', 'de', 'fr', 'es', 'it', 'pt', 'zh', 'ja', 'ko', 'ar')." + ), + parameters={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "Text to translate"}, + "targetLanguage": {"type": "string", "description": "Target language ISO code (e.g. 'en', 'de', 'fr')"}, + "sourceLanguage": {"type": "string", "description": "Source language ISO code (default: auto-detect)"}, + }, + "required": ["text", "targetLanguage"] + }, + readOnly=True + ) + + # ---- Phase 2: deleteFolder, renameFolder, moveFolder, copyFile, editFile ---- + + async def _deleteFolder(args: Dict[str, Any], context: Dict[str, Any]): + folderId = args.get("folderId", "") + recursive = args.get("recursive", False) + if not folderId: + return ToolResult(toolCallId="", toolName="deleteFolder", success=False, error="folderId is required") + try: + chatService = services.chat + result = chatService.interfaceDbComponent.deleteFolder(folderId, recursive=recursive) + summary = f"Deleted {result.get('deletedFolders', 1)} folder(s) and {result.get('deletedFiles', 0)} file(s)" + return ToolResult( + toolCallId="", toolName="deleteFolder", success=True, data=summary, + sideEvents=[{"type": "folderDeleted", "data": {"folderId": folderId, **result}}], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="deleteFolder", success=False, error=str(e)) + + async def _renameFolder(args: Dict[str, Any], context: Dict[str, Any]): + folderId = args.get("folderId", "") + newName = args.get("newName", "") + if not folderId or not newName: + return ToolResult(toolCallId="", toolName="renameFolder", success=False, error="folderId and newName are required") + try: + chatService = services.chat + chatService.interfaceDbComponent.renameFolder(folderId, newName) + return ToolResult( + toolCallId="", toolName="renameFolder", success=True, + data=f"Folder {folderId} renamed to '{newName}'", + sideEvents=[{"type": "folderUpdated", "data": {"folderId": folderId, "name": newName}}], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="renameFolder", success=False, error=str(e)) + + async def _moveFolder(args: Dict[str, Any], context: Dict[str, Any]): + folderId = args.get("folderId", "") + targetParentId = args.get("targetParentId") + if not folderId: + return ToolResult(toolCallId="", toolName="moveFolder", success=False, error="folderId is required") + try: + chatService = services.chat + chatService.interfaceDbComponent.moveFolder(folderId, targetParentId) + return ToolResult( + toolCallId="", toolName="moveFolder", success=True, + data=f"Folder {folderId} moved to {targetParentId or 'root'}", + sideEvents=[{"type": "folderUpdated", "data": {"folderId": folderId, "parentId": targetParentId}}], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="moveFolder", success=False, error=str(e)) + + async def _copyFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="copyFile", success=False, error="fileId is required") + try: + chatService = services.chat + copiedFile = chatService.interfaceDbComponent.copyFile( + fileId, + targetFolderId=args.get("targetFolderId"), + newFileName=args.get("newFileName"), + ) + return ToolResult( + toolCallId="", toolName="copyFile", success=True, + data=f"File copied as '{copiedFile.fileName}' (id: {copiedFile.id})", + sideEvents=[{ + "type": "fileCreated", + "data": {"fileId": copiedFile.id, "fileName": copiedFile.fileName, + "mimeType": copiedFile.mimeType, "fileSize": copiedFile.fileSize}, + }], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="copyFile", success=False, error=str(e)) + + async def _replaceInFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + oldText = args.get("oldText", "") + newText = args.get("newText", "") + replaceAll = args.get("replaceAll", False) + if not fileId or not oldText: + return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="fileId and oldText are required") + try: + chatService = services.chat + dbMgmt = chatService.interfaceDbComponent + file = dbMgmt.getFile(fileId) + if not file: + return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error=f"File {fileId} not found") + if not dbMgmt.isTextMimeType(file.mimeType): + return ToolResult( + toolCallId="", toolName="replaceInFile", success=False, + error=f"Cannot edit binary file ({file.mimeType}). Only text-based files are supported." + ) + rawData = dbMgmt.getFileData(fileId) + if not rawData: + return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="File has no content") + try: + oldContent = rawData.decode("utf-8") + except UnicodeDecodeError: + return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="File content is not valid UTF-8 text") + + count = oldContent.count(oldText) + if count == 0: + return ToolResult( + toolCallId="", toolName="replaceInFile", success=False, + error="oldText not found in file. Use readFile or searchInFileContent to verify the exact text." + ) + if count > 1 and not replaceAll: + return ToolResult( + toolCallId="", toolName="replaceInFile", success=False, + error=f"oldText found {count} times. Set replaceAll=true or provide more surrounding context to make it unique." + ) + + newContent = oldContent.replace(oldText, newText) if replaceAll else oldContent.replace(oldText, newText, 1) + + editId = str(_uuid.uuid4()) + label = f"all {count} occurrences" if replaceAll else "1 occurrence" + return ToolResult( + toolCallId="", toolName="replaceInFile", success=True, + data=f"Edit proposed for '{file.fileName}': replaced {label}. Waiting for user review.", + sideEvents=[{ + "type": "fileEditProposal", + "data": { + "id": editId, + "fileId": fileId, + "fileName": file.fileName, + "mimeType": file.mimeType, + "oldContent": oldContent, + "newContent": newContent, + }, + }], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error=str(e)) + + registry.register( + "deleteFolder", _deleteFolder, + description="Delete a folder from the local workspace. Set recursive=true to delete all contents.", + parameters={ + "type": "object", + "properties": { + "folderId": {"type": "string", "description": "The folder ID to delete"}, + "recursive": {"type": "boolean", "description": "If true, delete folder and all contents (files and subfolders). Default: false"}, + }, + "required": ["folderId"] + }, + readOnly=False + ) + + registry.register( + "renameFolder", _renameFolder, + description="Rename a folder in the local workspace.", + parameters={ + "type": "object", + "properties": { + "folderId": {"type": "string", "description": "The folder ID to rename"}, + "newName": {"type": "string", "description": "New folder name"}, + }, + "required": ["folderId", "newName"] + }, + readOnly=False + ) + + registry.register( + "moveFolder", _moveFolder, + description="Move a folder to a different parent in the local workspace.", + parameters={ + "type": "object", + "properties": { + "folderId": {"type": "string", "description": "The folder ID to move"}, + "targetParentId": {"type": "string", "description": "Target parent folder ID (null/omit for root)"}, + }, + "required": ["folderId"] + }, + readOnly=False + ) + + registry.register( + "copyFile", _copyFile, + description="Create an independent copy of a file in the local workspace.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to copy"}, + "targetFolderId": {"type": "string", "description": "Target folder for the copy (default: same folder)"}, + "newFileName": {"type": "string", "description": "New file name (default: same name, auto-numbered if duplicate)"}, + }, + "required": ["fileId"] + }, + readOnly=False + ) + + registry.register( + "replaceInFile", _replaceInFile, + description=( + "Replace specific text in an existing file. The edit is shown to the user for " + "review (accept/reject) before being applied. Provide enough surrounding context " + "in oldText to make the match unique (at least 2-3 lines). " + "Use readFile or searchInFileContent first to identify the exact text to replace." + ), + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to edit"}, + "oldText": {"type": "string", "description": "Exact text to find and replace (must be unique unless replaceAll=true)"}, + "newText": {"type": "string", "description": "The replacement text"}, + "replaceAll": {"type": "boolean", "description": "Replace all occurrences (default: false)"}, + }, + "required": ["fileId", "oldText", "newText"] + }, + readOnly=False + ) + diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/registerCore.py b/modules/serviceCenter/services/serviceAgent/coreTools/registerCore.py new file mode 100644 index 00000000..234da7d8 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/coreTools/registerCore.py @@ -0,0 +1,27 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Orchestrator: registers all core agent tools by delegating to domain modules.""" + +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +from modules.serviceCenter.services.serviceAgent.coreTools._workspaceTools import _registerWorkspaceTools +from modules.serviceCenter.services.serviceAgent.coreTools._connectionTools import _registerConnectionTools +from modules.serviceCenter.services.serviceAgent.coreTools._dataSourceTools import _registerDataSourceTools +from modules.serviceCenter.services.serviceAgent.coreTools._documentTools import _registerDocumentTools +from modules.serviceCenter.services.serviceAgent.coreTools._mediaTools import _registerMediaTools +from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import _registerFeatureSubAgentTools +from modules.serviceCenter.services.serviceAgent.coreTools._crossWorkflowTools import _registerCrossWorkflowTools + + +def registerCoreTools(registry: ToolRegistry, services): + """Register all built-in core tools on the agent ToolRegistry. + + Delegates to domain-specific modules under coreTools/. + """ + _registerWorkspaceTools(registry, services) + _registerConnectionTools(registry, services) + _registerDataSourceTools(registry, services) + _registerDocumentTools(registry, services) + _registerMediaTools(registry, services) + _registerFeatureSubAgentTools(registry, services) + _registerCrossWorkflowTools(registry, services) diff --git a/modules/serviceCenter/services/serviceAgent/datamodelAgent.py b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py index f682e705..e09e16b8 100644 --- a/modules/serviceCenter/services/serviceAgent/datamodelAgent.py +++ b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py @@ -88,6 +88,8 @@ class AgentConfig(BaseModel): maxRounds: int = Field(default=25, ge=1, le=100) maxCostCHF: Optional[float] = Field(default=None, ge=0.0) toolSet: str = Field(default="core") + initialToolboxes: List[str] = Field(default_factory=lambda: ["core"]) + availableToolboxes: List[str] = Field(default_factory=list) temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) diff --git a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py index b370b827..2d5a5dfb 100644 --- a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py +++ b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py @@ -14,6 +14,7 @@ from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop from modules.serviceCenter.services.serviceAgent.actionToolAdapter import ActionToolAdapter +from modules.serviceCenter.services.serviceAgent.coreTools import registerCoreTools from modules.serviceCenter.services.serviceBilling.mainServiceBilling import ( getService as getBillingService, InsufficientBalanceException, @@ -22,44 +23,6 @@ from modules.serviceCenter.services.serviceBilling.mainServiceBilling import ( logger = logging.getLogger(__name__) -_MAX_TOOL_RESULT_CHARS = 50_000 - -_BINARY_SIGNATURES = (b"%PDF", b"\x89PNG", b"\xff\xd8\xff", b"GIF8", b"PK\x03\x04", b"Rar!", b"\x1f\x8b") - - -def _resolveFileScope(fileId: str, context: dict) -> tuple: - """Resolve featureInstanceId and mandateId for a file from context or management DB. - - Returns (featureInstanceId, mandateId) — never None, always strings. - """ - fiId = context.get("featureInstanceId", "") or "" - mId = context.get("mandateId", "") or "" - if fiId and mId: - return fiId, mId - try: - from modules.datamodels.datamodelFiles import FileItem - from modules.interfaces.interfaceDbManagement import ComponentObjects - fm = ComponentObjects().db._loadRecord(FileItem, fileId) - if fm: - _get = (lambda k: fm.get(k, "")) if isinstance(fm, dict) else (lambda k: getattr(fm, k, "")) - fiId = fiId or str(_get("featureInstanceId") or "") - mId = mId or str(_get("mandateId") or "") - except Exception: - pass - return fiId, mId - - -def _looksLikeBinary(data: bytes, sampleSize: int = 1024) -> bool: - """Detect binary content by checking for magic bytes and non-printable char ratio.""" - if any(data[:8].startswith(sig) for sig in _BINARY_SIGNATURES): - return True - sample = data[:sampleSize] - if not sample: - return False - nonPrintable = sum(1 for b in sample if b < 0x09 or (0x0E <= b < 0x20 and b != 0x1B)) - return nonPrintable / len(sample) > 0.10 - - class _ServicesAdapter: """Adapter providing service access from (context, get_service).""" @@ -298,10 +261,10 @@ class AgentService: return prompt def _buildToolRegistry(self, config: AgentConfig) -> ToolRegistry: - """Build a tool registry with core tools and ActionToolAdapter tools.""" + """Build a tool registry with core tools, ActionToolAdapter tools, and toolbox-activated tools.""" registry = ToolRegistry() - _registerCoreTools(registry, self.services) + registerCoreTools(registry, self.services) try: from modules.workflows.processing.core.actionExecutor import ActionExecutor @@ -311,8 +274,41 @@ class AgentService: except Exception as e: logger.warning(f"Could not register action tools: {e}") + self._activateToolboxes(registry, config) + return registry + def _activateToolboxes(self, registry: ToolRegistry, config: AgentConfig) -> None: + """Activate toolboxes dynamically based on user connections and config.""" + try: + from modules.serviceCenter.services.serviceAgent.toolboxRegistry import getToolboxRegistry + tbRegistry = getToolboxRegistry() + + userConnections: List[str] = [] + try: + connectionService = self._getService("connection") + if connectionService and hasattr(connectionService, "getConnections"): + connections = connectionService.getConnections() or [] + userConnections = [c.get("authority", "") for c in connections if c.get("authority")] + except Exception as e: + logger.debug("Could not resolve user connections for toolbox activation: %s", e) + + activeToolboxes = tbRegistry.getActiveToolboxes(userConnections) + activatedIds = [tb.id for tb in activeToolboxes] + logger.info("Toolbox activation: connections=%s -> active toolboxes=%s", userConnections, activatedIds) + + for tb in activeToolboxes: + if tb.id == "workflow": + try: + from modules.serviceCenter.services.serviceAgent.workflowTools import getWorkflowToolDefinitions + for toolDef in getWorkflowToolDefinitions(): + registry.register(toolDef) + logger.info("Registered %d workflow tools from toolbox", len(getWorkflowToolDefinitions())) + except Exception as e: + logger.warning("Could not register workflow tools: %s", e) + except Exception as e: + logger.warning("Toolbox activation failed: %s", e) + async def _persistTrace(self, workflowId: str, summaryData: Dict[str, Any]): """Persist the agent trace and workflow artifacts in the knowledge store.""" try: @@ -501,2942 +497,3 @@ def _buildWorkflowHintItems( countLabel += f" (showing 10 newest)" items.insert(0, {"key": countLabel, "value": "use listWorkflowHistory to browse"}) return items - - -def _getOrCreateTempFolder(chatService) -> Optional[str]: - """Return the ID of the root-level 'Temp' folder, creating it if it doesn't exist.""" - try: - allFolders = chatService.interfaceDbComponent.listFolders() - tempFolder = next( - (f for f in allFolders - if f.get("name") == "Temp" and not f.get("parentId")), - None, - ) - if tempFolder: - return tempFolder.get("id") - newFolder = chatService.interfaceDbComponent.createFolder("Temp", parentId=None) - return newFolder.get("id") if newFolder else None - except Exception as e: - logger.warning(f"Could not get/create Temp folder: {e}") - return None - - -def _registerCoreTools(registry: ToolRegistry, services): - """Register built-in core tools: file operations, search, and folder management.""" - import uuid as _uuid - from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult - - # ---- Read-only tools ---- - - def _applyOffsetLimit(text: str, offset: int = None, limit: int = None) -> str: - """Apply line-based offset/limit to text content, returning numbered lines.""" - if offset is None and limit is None: - return None - lines = text.split("\n") - totalLines = len(lines) - startLine = max(0, (offset or 1) - 1) - endLine = min(totalLines, startLine + (limit or 200)) - selected = lines[startLine:endLine] - numbered = "\n".join(f"{i + startLine + 1}|{line}" for i, line in enumerate(selected)) - header = f"[Lines {startLine + 1}-{endLine} of {totalLines} total]\n" - return header + numbered - - async def _readFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - offset = args.get("offset") - limit = args.get("limit") - if not fileId: - return ToolResult(toolCallId="", toolName="readFile", success=False, error="fileId is required") - try: - knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None - - # 1) Knowledge Store: return already-extracted text chunks - if knowledgeService: - fileStatus = knowledgeService.getFileStatus(fileId) - if fileStatus == "indexed": - chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) - textChunks = [ - c for c in (chunks or []) - if c.get("contentType") != "image" and c.get("data") - ] - if textChunks: - assembled = "\n\n".join(c["data"] for c in textChunks) - chunked = _applyOffsetLimit(assembled, offset, limit) - if chunked is not None: - return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) - if len(assembled) > _MAX_TOOL_RESULT_CHARS: - assembled = assembled[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(assembled)}. Use offset/limit to read specific sections.]" - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=assembled, - ) - elif fileStatus in ("processing", "embedding", "extracted"): - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=f"[File {fileId} is currently being processed (status: {fileStatus}). Try again shortly.]", - ) - - # 2) Not indexed yet: try on-demand extraction - chatService = services.chat - fileInfo = chatService.getFileInfo(fileId) - if not fileInfo: - return ToolResult(toolCallId="", toolName="readFile", success=True, data="File not found.") - - fileName = fileInfo.get("fileName", fileId) - mimeType = fileInfo.get("mimeType", "") - - _BINARY_TYPES = ("application/pdf", "image/", "application/vnd.", "application/zip", - "application/x-zip", "application/x-tar", "application/x-7z", - "application/msword", "application/octet-stream", - "message/rfc822") - isBinary = any(mimeType.startswith(t) for t in _BINARY_TYPES) - - rawBytes = chatService.getFileData(fileId) - if not rawBytes: - return ToolResult(toolCallId="", toolName="readFile", success=True, data="File data not accessible.") - - if not isBinary: - isBinary = _looksLikeBinary(rawBytes) - - if isBinary: - try: - from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry, ChunkerRegistry - from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction - from modules.datamodels.datamodelExtraction import ExtractionOptions - - extracted = runExtraction( - ExtractorRegistry(), ChunkerRegistry(), - rawBytes, fileName, mimeType, ExtractionOptions(), - ) - - contentObjects = [] - for part in extracted.parts: - tg = (part.typeGroup or "").lower() - ct = "image" if tg == "image" else "text" - if not part.data or not part.data.strip(): - continue - contentObjects.append({ - "contentObjectId": part.id, - "contentType": ct, - "data": part.data, - "contextRef": { - "containerPath": fileName, - "location": part.label or "file", - **(part.metadata or {}), - }, - }) - - if contentObjects: - if knowledgeService: - try: - userId = context.get("userId", "") - _fiId, _mId = _resolveFileScope(fileId, context) - await knowledgeService.indexFile( - fileId=fileId, fileName=fileName, mimeType=mimeType, - userId=userId, contentObjects=contentObjects, - featureInstanceId=_fiId, - mandateId=_mId, - ) - except Exception: - pass - - joined = "" - if knowledgeService: - _chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) - _textChunks = [ - c for c in (_chunks or []) - if c.get("contentType") != "image" and c.get("data") - ] - if _textChunks: - joined = "\n\n".join(c["data"] for c in _textChunks) - if not joined: - textParts = [o["data"] for o in contentObjects if o["contentType"] != "image"] - joined = "\n\n".join(textParts) if textParts else "" - if joined: - chunked = _applyOffsetLimit(joined, offset, limit) - if chunked is not None: - return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) - if len(joined) > _MAX_TOOL_RESULT_CHARS: - joined = joined[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(joined)}. Use offset/limit to read specific sections.]" - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=joined, - ) - imgCount = sum(1 for o in contentObjects if o["contentType"] == "image") - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=f"[Extracted {len(contentObjects)} content objects from '{fileName}' " - f"({imgCount} images, no readable text). " - f"Use describeImage(fileId='{fileId}') to analyze visual content.]", - ) - except Exception as extractErr: - logger.warning(f"readFile extraction failed for {fileId} ({fileName}): {extractErr}") - - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=f"[Binary file: '{fileName}', type={mimeType}, size={len(rawBytes)} bytes. " - f"Text extraction not available. Use describeImage for images.]", - ) - - # 3) Text file: decode raw bytes - for encoding in ("utf-8", "utf-8-sig", "latin-1"): - try: - text = rawBytes.decode(encoding) - if text.strip(): - _fileNeedNeutralize = False - try: - from modules.datamodels.datamodelFiles import FileItem as _FI - from modules.interfaces.interfaceDbManagement import ComponentObjects as _CO - _fRec = _CO().db._loadRecord(_FI, fileId) - if _fRec: - _fG = (lambda k, d=None: _fRec.get(k, d)) if isinstance(_fRec, dict) else (lambda k, d=None: getattr(_fRec, k, d)) - _fileNeedNeutralize = bool(_fG("neutralize", False)) - except Exception: - pass - if _fileNeedNeutralize: - try: - _nSvc = services.getService("neutralization") if hasattr(services, "getService") else None - if _nSvc and hasattr(_nSvc, 'processTextAsync'): - _nResult = await _nSvc.processTextAsync(text, fileId) - if _nResult and _nResult.get("neutralized_text"): - text = _nResult["neutralized_text"] - logger.debug(f"readFile: neutralized text for file {fileId}") - else: - logger.warning(f"readFile: neutralization failed for file {fileId}, blocking text (fail-safe)") - return ToolResult(toolCallId="", toolName="readFile", success=True, - data="[File requires neutralization but neutralization failed. Content blocked for data protection.]") - else: - logger.warning(f"readFile: neutralization required but service unavailable for file {fileId}") - return ToolResult(toolCallId="", toolName="readFile", success=True, - data="[File requires neutralization but service unavailable. Content blocked for data protection.]") - except Exception as _nErr: - logger.error(f"readFile: neutralization error for file {fileId}: {_nErr}") - return ToolResult(toolCallId="", toolName="readFile", success=True, - data="[File requires neutralization but an error occurred. Content blocked for data protection.]") - chunked = _applyOffsetLimit(text, offset, limit) - if chunked is not None: - return ToolResult(toolCallId="", toolName="readFile", success=True, data=chunked) - if len(text) > _MAX_TOOL_RESULT_CHARS: - text = text[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(text)}. Use offset/limit to read specific sections.]" - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data=text, - ) - except (UnicodeDecodeError, ValueError): - continue - - return ToolResult( - toolCallId="", toolName="readFile", success=True, - data="File is empty or could not be decoded.", - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="readFile", success=False, error=str(e)) - - async def _listFiles(args: Dict[str, Any], context: Dict[str, Any]): - try: - chatService = services.chat - files = chatService.listFiles( - folderId=args.get("folderId"), - tags=args.get("tags"), - search=args.get("search"), - ) - fileList = "\n".join( - f"- {f.get('fileName', 'unknown')} (id: {f.get('id', '?')}, " - f"type: {f.get('mimeType', '?')}, size: {f.get('fileSize', '?')}, " - f"tags: {f.get('tags', [])}, status: {f.get('status', 'n/a')})" - for f in files - ) if files else "No files found." - return ToolResult(toolCallId="", toolName="listFiles", success=True, data=fileList) - except Exception as e: - return ToolResult(toolCallId="", toolName="listFiles", success=False, error=str(e)) - - async def _searchInFileContent(args: Dict[str, Any], context: Dict[str, Any]): - import re as _re - fileId = args.get("fileId", "") - query = args.get("query", "") - contextLines = args.get("contextLines", 2) - if not fileId or not query: - return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error="fileId and query are required") - try: - chatService = services.chat - rawBytes = chatService.getFileData(fileId) - if not rawBytes: - return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error="File data not accessible") - try: - content = rawBytes.decode("utf-8") - except UnicodeDecodeError: - content = rawBytes.decode("latin-1", errors="replace") - - lines = content.split("\n") - pattern = _re.compile(_re.escape(query), _re.IGNORECASE) - matches = [] - for i, line in enumerate(lines): - if pattern.search(line): - start = max(0, i - contextLines) - end = min(len(lines), i + contextLines + 1) - snippet = "\n".join(f"{j + 1}|{lines[j]}" for j in range(start, end)) - matches.append(snippet) - - if not matches: - return ToolResult(toolCallId="", toolName="searchInFileContent", success=True, - data=f"No matches for '{query}' in file.") - - shown = matches[:20] - resultText = f"Found {len(matches)} match(es) for '{query}':\n\n" + "\n---\n".join(shown) - if len(matches) > 20: - resultText += f"\n\n... and {len(matches) - 20} more matches" - return ToolResult(toolCallId="", toolName="searchInFileContent", success=True, data=resultText) - except Exception as e: - return ToolResult(toolCallId="", toolName="searchInFileContent", success=False, error=str(e)) - - async def _listFolders(args: Dict[str, Any], context: Dict[str, Any]): - try: - chatService = services.chat - folders = chatService.listFolders(parentId=args.get("parentId")) - folderList = "\n".join( - f"- {f.get('name', 'unnamed')} (id: {f.get('id', '?')})" - for f in folders - ) if folders else "No folders found." - return ToolResult(toolCallId="", toolName="listFolders", success=True, data=folderList) - except Exception as e: - return ToolResult(toolCallId="", toolName="listFolders", success=False, error=str(e)) - - async def _webSearch(args: Dict[str, Any], context: Dict[str, Any]): - query = args.get("query", "") - if not query: - return ToolResult(toolCallId="", toolName="webSearch", success=False, error="query is required") - try: - webService = services.getService("web") - result = await webService.performWebResearch( - prompt=query, - urls=[], - country=None, - language=args.get("language"), - ) - summary = result.get("summary", "") if isinstance(result, dict) else str(result) - return ToolResult( - toolCallId="", toolName="webSearch", success=True, - data=summary or str(result) - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="webSearch", success=False, error=str(e)) - - # ---- Write tools ---- - - async def _tagFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - tags = args.get("tags", []) - if not fileId: - return ToolResult(toolCallId="", toolName="tagFile", success=False, error="fileId is required") - try: - chatService = services.chat - chatService.interfaceDbComponent.updateFile(fileId, {"tags": tags}) - return ToolResult( - toolCallId="", toolName="tagFile", success=True, - data=f"Tags updated to {tags} for file {fileId}" - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="tagFile", success=False, error=str(e)) - - async def _moveFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - targetFolderId = args.get("targetFolderId") - if not fileId: - return ToolResult(toolCallId="", toolName="moveFile", success=False, error="fileId is required") - try: - chatService = services.chat - chatService.interfaceDbComponent.updateFile(fileId, {"folderId": targetFolderId}) - return ToolResult( - toolCallId="", toolName="moveFile", success=True, - data=f"File {fileId} moved to folder {targetFolderId or 'root'}" - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="moveFile", success=False, error=str(e)) - - async def _createFolder(args: Dict[str, Any], context: Dict[str, Any]): - name = args.get("name", "") - if not name: - return ToolResult(toolCallId="", toolName="createFolder", success=False, error="name is required") - try: - chatService = services.chat - folder = chatService.createFolder(name=name, parentId=args.get("parentId")) - return ToolResult( - toolCallId="", toolName="createFolder", success=True, - data=f"Folder '{name}' created (id: {folder.get('id', '?')})" - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="createFolder", success=False, error=str(e)) - - async def _writeFile(args: Dict[str, Any], context: Dict[str, Any]): - content = args.get("content", "") - mode = args.get("mode", "create") - fileId = args.get("fileId", "") - name = args.get("name", "") - - if not content: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error="content is required") - - try: - chatService = services.chat - dbMgmt = chatService.interfaceDbComponent - - if mode == "append": - if not fileId: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error="fileId is required for mode=append") - file = dbMgmt.getFile(fileId) - if not file: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error=f"File {fileId} not found") - existingData = dbMgmt.getFileData(fileId) or b"" - try: - existingText = existingData.decode("utf-8") - except UnicodeDecodeError: - existingText = existingData.decode("latin-1", errors="replace") - newContent = existingText + content - dbMgmt.updateFileData(fileId, newContent.encode("utf-8")) - dbMgmt.updateFile(fileId, {"fileSize": len(newContent.encode("utf-8"))}) - return ToolResult( - toolCallId="", toolName="writeFile", success=True, - data=f"Appended {len(content)} chars to '{file.fileName}' (id: {fileId}, total: {len(newContent)} chars)", - sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": file.fileName}}], - ) - - if mode == "overwrite": - if not fileId: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error="fileId is required for mode=overwrite") - file = dbMgmt.getFile(fileId) - if not file: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error=f"File {fileId} not found") - dbMgmt.updateFileData(fileId, content.encode("utf-8")) - dbMgmt.updateFile(fileId, {"fileSize": len(content.encode("utf-8"))}) - return ToolResult( - toolCallId="", toolName="writeFile", success=True, - data=f"Overwritten '{file.fileName}' (id: {fileId}, {len(content)} chars)", - sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": file.fileName}}], - ) - - # mode == "create" (default) - if not name: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error="name is required for mode=create") - fileItem, _ = dbMgmt.saveUploadedFile(content.encode("utf-8"), name) - fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") - if fiId: - dbMgmt.updateFile(fileItem.id, {"featureInstanceId": fiId}) - if args.get("folderId"): - dbMgmt.updateFile(fileItem.id, {"folderId": args["folderId"]}) - if args.get("tags"): - dbMgmt.updateFile(fileItem.id, {"tags": args["tags"]}) - return ToolResult( - toolCallId="", toolName="writeFile", success=True, - data=f"File '{name}' created (id: {fileItem.id})", - sideEvents=[{ - "type": "fileCreated", - "data": { - "fileId": fileItem.id, - "fileName": name, - "mimeType": fileItem.mimeType, - "fileSize": fileItem.fileSize, - }, - }], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="writeFile", success=False, error=str(e)) - - # ---- Register all tools ---- - - registry.register( - "readFile", _readFile, - description=( - "Read the content of a file. Returns full content by default. " - "For large files, use offset and limit to read specific line ranges. " - "When truncated, the response tells the total line count so you can paginate." - ), - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to read"}, - "offset": {"type": "integer", "description": "Start reading from this line number (1-based). Omit for full file."}, - "limit": {"type": "integer", "description": "Max number of lines to return (default: all). Use with offset for chunked reading."}, - }, - "required": ["fileId"] - }, - readOnly=True - ) - - registry.register( - "listFiles", _listFiles, - description=( - "List files in the local workspace. Filter by folder, tags, or search term. " - "For external data sources, use browseDataSource instead." - ), - parameters={ - "type": "object", - "properties": { - "folderId": {"type": "string", "description": "Filter by folder ID"}, - "tags": {"type": "array", "items": {"type": "string"}, "description": "Filter by tags (any match)"}, - "search": {"type": "string", "description": "Search in file names and descriptions"}, - } - }, - readOnly=True - ) - - registry.register( - "searchInFileContent", _searchInFileContent, - description=( - "Search for text within a file's content. Returns matching lines with context. " - "Case-insensitive. Use to locate specific text before using replaceInFile, " - "or to find relevant sections in a large file before reading with offset/limit." - ), - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to search in"}, - "query": {"type": "string", "description": "Text to search for (case-insensitive)"}, - "contextLines": {"type": "integer", "description": "Number of context lines around each match (default: 2)"}, - }, - "required": ["fileId", "query"] - }, - readOnly=True - ) - - registry.register( - "listFolders", _listFolders, - description="List folders in the local workspace. For external data sources, use browseDataSource instead.", - parameters={ - "type": "object", - "properties": { - "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, - } - }, - readOnly=True - ) - - registry.register( - "webSearch", _webSearch, - description="Search the web for general information. Use readUrl to fetch content from a known URL instead.", - parameters={ - "type": "object", - "properties": {"query": {"type": "string", "description": "Search query"}}, - "required": ["query"] - }, - readOnly=True - ) - - registry.register( - "tagFile", _tagFile, - description="Set or update tags on a file for categorization and filtering via listFiles.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID"}, - "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags to set"}, - }, - "required": ["fileId", "tags"] - }, - readOnly=False - ) - - registry.register( - "moveFile", _moveFile, - description="Move a file to a different folder in the local workspace.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to move"}, - "targetFolderId": {"type": "string", "description": "Target folder ID (null for root)"}, - }, - "required": ["fileId"] - }, - readOnly=False - ) - - registry.register( - "createFolder", _createFolder, - description="Create a new folder in the local workspace.", - parameters={ - "type": "object", - "properties": { - "name": {"type": "string", "description": "Folder name"}, - "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, - }, - "required": ["name"] - }, - readOnly=False - ) - - registry.register( - "writeFile", _writeFile, - description=( - "Create, append, or overwrite a file. Modes:\n" - "- create (default): create a new file (name required).\n" - "- append: append content to an existing file (fileId required). " - "Use for large content that exceeds a single tool call (~8000 chars per call).\n" - "- overwrite: replace entire file content (fileId required)." - ), - parameters={ - "type": "object", - "properties": { - "name": {"type": "string", "description": "File name (required for mode=create)"}, - "content": {"type": "string", "description": "Content to write/append"}, - "mode": {"type": "string", "enum": ["create", "append", "overwrite"], "description": "Write mode (default: create)"}, - "fileId": {"type": "string", "description": "File ID (required for mode=append/overwrite)"}, - "folderId": {"type": "string", "description": "Target folder ID (mode=create only)"}, - "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags (mode=create only)"}, - }, - "required": ["content"] - }, - readOnly=False - ) - - # ---- Phase 1: deleteFile, renameFile, readUrl, translateText ---- - - async def _deleteFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="deleteFile", success=False, error="fileId is required") - try: - chatService = services.chat - file = chatService.interfaceDbComponent.getFile(fileId) - if not file: - return ToolResult(toolCallId="", toolName="deleteFile", success=False, error=f"File {fileId} not found") - fileName = file.fileName - try: - knowledgeService = services.getService("knowledge") - if knowledgeService and hasattr(knowledgeService, "removeFile"): - knowledgeService.removeFile(fileId) - except Exception: - pass - chatService.interfaceDbComponent.deleteFile(fileId) - return ToolResult( - toolCallId="", toolName="deleteFile", success=True, - data=f"File '{fileName}' (id: {fileId}) deleted", - sideEvents=[{"type": "fileDeleted", "data": {"fileId": fileId, "fileName": fileName}}], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="deleteFile", success=False, error=str(e)) - - async def _renameFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - newName = args.get("newName", "") - if not fileId or not newName: - return ToolResult(toolCallId="", toolName="renameFile", success=False, error="fileId and newName are required") - try: - chatService = services.chat - chatService.interfaceDbComponent.updateFile(fileId, {"fileName": newName}) - return ToolResult( - toolCallId="", toolName="renameFile", success=True, - data=f"File {fileId} renamed to '{newName}'", - sideEvents=[{"type": "fileUpdated", "data": {"fileId": fileId, "fileName": newName}}], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="renameFile", success=False, error=str(e)) - - async def _readUrl(args: Dict[str, Any], context: Dict[str, Any]): - url = args.get("url", "") - if not url: - return ToolResult(toolCallId="", toolName="readUrl", success=False, error="url is required") - try: - webService = services.getService("web") - result = await webService._performWebCrawl( - instruction="Extract all content from this page", - urls=[url], - maxDepth=1, - maxWidth=1, - ) - if isinstance(result, list) and result: - content = "\n\n".join( - item.get("content", "") or item.get("text", "") or str(item) - for item in result if item - ) - elif isinstance(result, dict): - content = result.get("content", "") or result.get("summary", "") or str(result) - else: - content = str(result) if result else "No content retrieved" - _MAX = 30000 - if len(content) > _MAX: - content = content[:_MAX] + f"\n\n... (truncated at {_MAX} chars)" - return ToolResult(toolCallId="", toolName="readUrl", success=True, data=content) - except Exception as e: - return ToolResult(toolCallId="", toolName="readUrl", success=False, error=str(e)) - - async def _translateText(args: Dict[str, Any], context: Dict[str, Any]): - text = args.get("text", "") - targetLanguage = args.get("targetLanguage", "") - if not text or not targetLanguage: - return ToolResult(toolCallId="", toolName="translateText", success=False, error="text and targetLanguage are required") - try: - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - mandateId = context.get("mandateId", "") - voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) - sourceLanguage = args.get("sourceLanguage", "auto") - result = await voiceInterface.translateText(text, sourceLanguage=sourceLanguage, targetLanguage=targetLanguage) - if result and result.get("success"): - translated = result.get("translated_text", "") - return ToolResult(toolCallId="", toolName="translateText", success=True, data=translated) - return ToolResult(toolCallId="", toolName="translateText", success=False, error=result.get("error", "Translation failed")) - except Exception as e: - return ToolResult(toolCallId="", toolName="translateText", success=False, error=str(e)) - - registry.register( - "deleteFile", _deleteFile, - description="Permanently delete a file from the local workspace.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to delete"}, - }, - "required": ["fileId"] - }, - readOnly=False - ) - - registry.register( - "renameFile", _renameFile, - description="Rename a file in the local workspace. Include the file extension in the new name.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to rename"}, - "newName": {"type": "string", "description": "New file name including extension"}, - }, - "required": ["fileId", "newName"] - }, - readOnly=False - ) - - registry.register( - "readUrl", _readUrl, - description=( - "Read and extract content from a specific URL. " - "Use when the user provides a specific URL to read, or when you need to fetch content from a known web page. " - "For general information searches, use webSearch instead." - ), - parameters={ - "type": "object", - "properties": { - "url": {"type": "string", "description": "The URL to read"}, - }, - "required": ["url"] - }, - readOnly=True - ) - - registry.register( - "translateText", _translateText, - description=( - "Translate text to a target language using Google Cloud Translation. " - "More efficient than AI translation for large text volumes. " - "Use ISO language codes (e.g. 'en', 'de', 'fr', 'es', 'it', 'pt', 'zh', 'ja', 'ko', 'ar')." - ), - parameters={ - "type": "object", - "properties": { - "text": {"type": "string", "description": "Text to translate"}, - "targetLanguage": {"type": "string", "description": "Target language ISO code (e.g. 'en', 'de', 'fr')"}, - "sourceLanguage": {"type": "string", "description": "Source language ISO code (default: auto-detect)"}, - }, - "required": ["text", "targetLanguage"] - }, - readOnly=True - ) - - # ---- Phase 2: deleteFolder, renameFolder, moveFolder, copyFile, editFile ---- - - async def _deleteFolder(args: Dict[str, Any], context: Dict[str, Any]): - folderId = args.get("folderId", "") - recursive = args.get("recursive", False) - if not folderId: - return ToolResult(toolCallId="", toolName="deleteFolder", success=False, error="folderId is required") - try: - chatService = services.chat - result = chatService.interfaceDbComponent.deleteFolder(folderId, recursive=recursive) - summary = f"Deleted {result.get('deletedFolders', 1)} folder(s) and {result.get('deletedFiles', 0)} file(s)" - return ToolResult( - toolCallId="", toolName="deleteFolder", success=True, data=summary, - sideEvents=[{"type": "folderDeleted", "data": {"folderId": folderId, **result}}], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="deleteFolder", success=False, error=str(e)) - - async def _renameFolder(args: Dict[str, Any], context: Dict[str, Any]): - folderId = args.get("folderId", "") - newName = args.get("newName", "") - if not folderId or not newName: - return ToolResult(toolCallId="", toolName="renameFolder", success=False, error="folderId and newName are required") - try: - chatService = services.chat - chatService.interfaceDbComponent.renameFolder(folderId, newName) - return ToolResult( - toolCallId="", toolName="renameFolder", success=True, - data=f"Folder {folderId} renamed to '{newName}'", - sideEvents=[{"type": "folderUpdated", "data": {"folderId": folderId, "name": newName}}], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="renameFolder", success=False, error=str(e)) - - async def _moveFolder(args: Dict[str, Any], context: Dict[str, Any]): - folderId = args.get("folderId", "") - targetParentId = args.get("targetParentId") - if not folderId: - return ToolResult(toolCallId="", toolName="moveFolder", success=False, error="folderId is required") - try: - chatService = services.chat - chatService.interfaceDbComponent.moveFolder(folderId, targetParentId) - return ToolResult( - toolCallId="", toolName="moveFolder", success=True, - data=f"Folder {folderId} moved to {targetParentId or 'root'}", - sideEvents=[{"type": "folderUpdated", "data": {"folderId": folderId, "parentId": targetParentId}}], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="moveFolder", success=False, error=str(e)) - - async def _copyFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="copyFile", success=False, error="fileId is required") - try: - chatService = services.chat - copiedFile = chatService.interfaceDbComponent.copyFile( - fileId, - targetFolderId=args.get("targetFolderId"), - newFileName=args.get("newFileName"), - ) - return ToolResult( - toolCallId="", toolName="copyFile", success=True, - data=f"File copied as '{copiedFile.fileName}' (id: {copiedFile.id})", - sideEvents=[{ - "type": "fileCreated", - "data": {"fileId": copiedFile.id, "fileName": copiedFile.fileName, - "mimeType": copiedFile.mimeType, "fileSize": copiedFile.fileSize}, - }], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="copyFile", success=False, error=str(e)) - - async def _replaceInFile(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - oldText = args.get("oldText", "") - newText = args.get("newText", "") - replaceAll = args.get("replaceAll", False) - if not fileId or not oldText: - return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="fileId and oldText are required") - try: - chatService = services.chat - dbMgmt = chatService.interfaceDbComponent - file = dbMgmt.getFile(fileId) - if not file: - return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error=f"File {fileId} not found") - if not dbMgmt.isTextMimeType(file.mimeType): - return ToolResult( - toolCallId="", toolName="replaceInFile", success=False, - error=f"Cannot edit binary file ({file.mimeType}). Only text-based files are supported." - ) - rawData = dbMgmt.getFileData(fileId) - if not rawData: - return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="File has no content") - try: - oldContent = rawData.decode("utf-8") - except UnicodeDecodeError: - return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error="File content is not valid UTF-8 text") - - count = oldContent.count(oldText) - if count == 0: - return ToolResult( - toolCallId="", toolName="replaceInFile", success=False, - error="oldText not found in file. Use readFile or searchInFileContent to verify the exact text." - ) - if count > 1 and not replaceAll: - return ToolResult( - toolCallId="", toolName="replaceInFile", success=False, - error=f"oldText found {count} times. Set replaceAll=true or provide more surrounding context to make it unique." - ) - - newContent = oldContent.replace(oldText, newText) if replaceAll else oldContent.replace(oldText, newText, 1) - - editId = str(_uuid.uuid4()) - label = f"all {count} occurrences" if replaceAll else "1 occurrence" - return ToolResult( - toolCallId="", toolName="replaceInFile", success=True, - data=f"Edit proposed for '{file.fileName}': replaced {label}. Waiting for user review.", - sideEvents=[{ - "type": "fileEditProposal", - "data": { - "id": editId, - "fileId": fileId, - "fileName": file.fileName, - "mimeType": file.mimeType, - "oldContent": oldContent, - "newContent": newContent, - }, - }], - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="replaceInFile", success=False, error=str(e)) - - registry.register( - "deleteFolder", _deleteFolder, - description="Delete a folder from the local workspace. Set recursive=true to delete all contents.", - parameters={ - "type": "object", - "properties": { - "folderId": {"type": "string", "description": "The folder ID to delete"}, - "recursive": {"type": "boolean", "description": "If true, delete folder and all contents (files and subfolders). Default: false"}, - }, - "required": ["folderId"] - }, - readOnly=False - ) - - registry.register( - "renameFolder", _renameFolder, - description="Rename a folder in the local workspace.", - parameters={ - "type": "object", - "properties": { - "folderId": {"type": "string", "description": "The folder ID to rename"}, - "newName": {"type": "string", "description": "New folder name"}, - }, - "required": ["folderId", "newName"] - }, - readOnly=False - ) - - registry.register( - "moveFolder", _moveFolder, - description="Move a folder to a different parent in the local workspace.", - parameters={ - "type": "object", - "properties": { - "folderId": {"type": "string", "description": "The folder ID to move"}, - "targetParentId": {"type": "string", "description": "Target parent folder ID (null/omit for root)"}, - }, - "required": ["folderId"] - }, - readOnly=False - ) - - registry.register( - "copyFile", _copyFile, - description="Create an independent copy of a file in the local workspace.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to copy"}, - "targetFolderId": {"type": "string", "description": "Target folder for the copy (default: same folder)"}, - "newFileName": {"type": "string", "description": "New file name (default: same name, auto-numbered if duplicate)"}, - }, - "required": ["fileId"] - }, - readOnly=False - ) - - registry.register( - "replaceInFile", _replaceInFile, - description=( - "Replace specific text in an existing file. The edit is shown to the user for " - "review (accept/reject) before being applied. Provide enough surrounding context " - "in oldText to make the match unique (at least 2-3 lines). " - "Use readFile or searchInFileContent first to identify the exact text to replace." - ), - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID to edit"}, - "oldText": {"type": "string", "description": "Exact text to find and replace (must be unique unless replaceAll=true)"}, - "newText": {"type": "string", "description": "The replacement text"}, - "replaceAll": {"type": "boolean", "description": "Replace all occurrences (default: false)"}, - }, - "required": ["fileId", "oldText", "newText"] - }, - readOnly=False - ) - - # ---- Connection tools (external data sources) ---- - - def _buildResolverDb(): - """Build a DB adapter that ConnectorResolver can use to load UserConnections. - interfaceDbApp has getUserConnectionById; ConnectorResolver expects getUserConnection.""" - chatService = services.chat - appIf = getattr(chatService, "interfaceDbApp", None) - if appIf and hasattr(appIf, "getUserConnectionById"): - class _Adapter: - def __init__(self, app): - self._app = app - def getUserConnection(self, connectionId: str): - return self._app.getUserConnectionById(connectionId) - return _Adapter(appIf) - return getattr(chatService, "interfaceDbComponent", None) - - async def _listConnections(args: Dict[str, Any], context: Dict[str, Any]): - try: - chatService = services.chat - connections = chatService.getUserConnections() if hasattr(chatService, "getUserConnections") else [] - if not connections: - return ToolResult(toolCallId="", toolName="listConnections", success=True, data="No connections available.") - lines = [] - for conn in connections: - connId = conn.get("id", "?") if isinstance(conn, dict) else getattr(conn, "id", "?") - authority = conn.get("authority", "?") if isinstance(conn, dict) else getattr(conn, "authority", "?") - email = conn.get("externalEmail", "") if isinstance(conn, dict) else getattr(conn, "externalEmail", "") - lines.append(f"- {authority} ({email}) id: {connId}") - return ToolResult(toolCallId="", toolName="listConnections", success=True, data="\n".join(lines)) - except Exception as e: - return ToolResult(toolCallId="", toolName="listConnections", success=False, error=str(e)) - - async def _uploadToExternal(args: Dict[str, Any], context: Dict[str, Any]): - connectionId = args.get("connectionId", "") - service = args.get("service", "") - path = args.get("path", "") - fileId = args.get("fileId", "") - if not connectionId or not service or not path or not fileId: - return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error="connectionId, service, path, and fileId are required") - try: - from modules.connectors.connectorResolver import ConnectorResolver - resolver = ConnectorResolver( - services.getService("security"), - _buildResolverDb(), - ) - adapter = await resolver.resolveService(connectionId, service) - chatService = services.chat - fileContent = chatService.getFileContent(fileId) - if not fileContent: - return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error="File not found") - fileData = fileContent.get("data", b"") if isinstance(fileContent, dict) else b"" - if isinstance(fileData, str): - fileData = fileData.encode("utf-8") - fileName = fileContent.get("fileName", "file") if isinstance(fileContent, dict) else "file" - result = await adapter.upload(path, fileData, fileName) - return ToolResult(toolCallId="", toolName="uploadToExternal", success=True, data=str(result)) - except Exception as e: - return ToolResult(toolCallId="", toolName="uploadToExternal", success=False, error=str(e)) - - async def _sendMail(args: Dict[str, Any], context: Dict[str, Any]): - import base64 as _b64 - - connectionId = args.get("connectionId", "") - to = args.get("to", []) - subject = args.get("subject", "") - body = args.get("body", "") - bodyType = "HTML" if args.get("bodyType", "text").lower() == "html" else "Text" - draft = args.get("draft", False) - attachmentFileIds = args.get("attachmentFileIds") or [] - - if not connectionId or not to or not subject: - return ToolResult(toolCallId="", toolName="sendMail", success=False, error="connectionId, to, and subject are required") - try: - graphAttachments: List[Dict[str, Any]] = [] - if attachmentFileIds: - chatService = services.chat - dbMgmt = chatService.interfaceDbComponent - for fid in attachmentFileIds: - fileRow = dbMgmt.getFile(fid) - if not fileRow: - return ToolResult(toolCallId="", toolName="sendMail", success=False, error=f"Attachment file not found: {fid}") - rawBytes = dbMgmt.getFileData(fid) - if not rawBytes: - return ToolResult(toolCallId="", toolName="sendMail", success=False, error=f"Attachment file has no data: {fid}") - graphAttachments.append({ - "name": fileRow.fileName, - "contentBytes": _b64.b64encode(rawBytes).decode("ascii"), - "contentType": getattr(fileRow, "mimeType", "application/octet-stream"), - }) - - from modules.connectors.connectorResolver import ConnectorResolver - resolver = ConnectorResolver( - services.getService("security"), - _buildResolverDb(), - ) - adapter = await resolver.resolveService(connectionId, "outlook") - - if draft and hasattr(adapter, "createDraft"): - result = await adapter.createDraft( - to=to, subject=subject, body=body, bodyType=bodyType, - cc=args.get("cc"), attachments=graphAttachments or None, - ) - return ToolResult(toolCallId="", toolName="sendMail", success=True, data=str(result)) - - if hasattr(adapter, "sendMail"): - result = await adapter.sendMail( - to=to, subject=subject, body=body, bodyType=bodyType, - cc=args.get("cc"), attachments=graphAttachments or None, - ) - return ToolResult(toolCallId="", toolName="sendMail", success=True, data=str(result)) - return ToolResult(toolCallId="", toolName="sendMail", success=False, error="Mail not supported by this adapter") - except Exception as e: - return ToolResult(toolCallId="", toolName="sendMail", success=False, error=str(e)) - - _connToolParams = { - "connectionId": {"type": "string", "description": "UserConnection ID"}, - "service": {"type": "string", "description": "Service name (sharepoint, outlook, drive, etc.)"}, - } - - registry.register( - "listConnections", _listConnections, - description="List the user's external connections (SharePoint, OneDrive, Outlook, etc.) and their IDs. Use with browseDataSource/uploadToExternal.", - parameters={"type": "object", "properties": {}}, - readOnly=True, - ) - - registry.register( - "uploadToExternal", _uploadToExternal, - description=( - "Upload a local file to an external storage via connectionId+service. " - "Use listConnections to find available connections." - ), - parameters={ - "type": "object", - "properties": { - **_connToolParams, - "path": {"type": "string", "description": "Destination path on the external service"}, - "fileId": {"type": "string", "description": "Local file ID to upload"}, - }, - "required": ["connectionId", "service", "path", "fileId"], - }, - readOnly=False, - ) - - registry.register( - "sendMail", _sendMail, - description=( - "Send or draft an email via a connected mail service (Outlook). " - "Supports HTML body and file attachments from the workspace. " - "Set draft=true to save as draft without sending. " - "Use listConnections to find the connectionId." - ), - parameters={ - "type": "object", - "properties": { - "connectionId": {"type": "string", "description": "UserConnection ID"}, - "to": {"type": "array", "items": {"type": "string"}, "description": "Recipient email addresses"}, - "subject": {"type": "string", "description": "Email subject"}, - "body": {"type": "string", "description": "Email body — plain text or HTML markup"}, - "bodyType": {"type": "string", "enum": ["text", "html"], "description": "Body format: 'text' (default) or 'html'"}, - "cc": {"type": "array", "items": {"type": "string"}, "description": "CC addresses"}, - "attachmentFileIds": { - "type": "array", "items": {"type": "string"}, - "description": "File IDs from the workspace to attach (use listFiles to find IDs)", - }, - "draft": {"type": "boolean", "description": "If true, save as draft in Drafts folder instead of sending"}, - }, - "required": ["connectionId", "to", "subject", "body"], - }, - readOnly=False, - ) - - # ---- DataSource convenience tools ---- - _SOURCE_TYPE_TO_SERVICE = { - "sharepointFolder": "sharepoint", - "onedriveFolder": "onedrive", - "outlookFolder": "outlook", - "googleDriveFolder": "drive", - "gmailFolder": "gmail", - "ftpFolder": "files", - "clickupList": "clickup", - } - - async def _resolveDataSource(dsId: str): - """Resolve a DataSource record and return (connectionId, service, path, neutralize) or raise.""" - chatService = services.chat - ds = chatService.getDataSource(dsId) if hasattr(chatService, "getDataSource") else None - if not ds: - raise ValueError(f"DataSource '{dsId}' not found") - connectionId = ds.get("connectionId", "") - sourceType = ds.get("sourceType", "") - path = ds.get("path", "/") - label = ds.get("label", "") - neutralize = bool(ds.get("neutralize", False)) - service = _SOURCE_TYPE_TO_SERVICE.get(sourceType, sourceType) - if not connectionId: - raise ValueError(f"DataSource '{dsId}' has no connectionId") - logger.info(f"Resolved DataSource '{dsId}' ({label}): sourceType={sourceType}, service={service}, connectionId={connectionId}, path={path[:80]}, neutralize={neutralize}") - return connectionId, service, path, neutralize - - _MAIL_SERVICES = {"outlook", "gmail"} - - async def _browseDataSource(args: Dict[str, Any], context: Dict[str, Any]): - dsId = args.get("dataSourceId", "") - subPath = args.get("subPath", "") - directConnId = args.get("connectionId", "") - directService = args.get("service", "") - if not dsId and not (directConnId and directService): - return ToolResult(toolCallId="", toolName="browseDataSource", success=False, - error="Provide either dataSourceId OR connectionId+service") - try: - if dsId: - connectionId, service, basePath, _neutralize = await _resolveDataSource(dsId) - else: - connectionId, service, basePath = directConnId, directService, args.get("path", "/") - if subPath: - if subPath.startswith("/"): - browsePath = subPath - else: - browsePath = f"{basePath.rstrip('/')}/{subPath}" - else: - browsePath = basePath - from modules.connectors.connectorResolver import ConnectorResolver - resolver = ConnectorResolver( - services.getService("security"), - _buildResolverDb(), - ) - adapter = await resolver.resolveService(connectionId, service) - entries = await adapter.browse(browsePath, filter=args.get("filter")) - if not entries: - return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data="Empty directory.") - lines = [] - for e in entries: - prefix = "[DIR]" if e.isFolder else "[FILE]" - sizeInfo = f" ({e.size} bytes)" if e.size else "" - lines.append(f"- {prefix} {e.name}{sizeInfo} path: {e.path}") - result = "\n".join(lines) - if service in _MAIL_SERVICES: - result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." - return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data=result) - except Exception as e: - return ToolResult(toolCallId="", toolName="browseDataSource", success=False, error=str(e)) - - async def _searchDataSource(args: Dict[str, Any], context: Dict[str, Any]): - dsId = args.get("dataSourceId", "") - directConnId = args.get("connectionId", "") - directService = args.get("service", "") - query = args.get("query", "") - if not query: - return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error="query is required") - if not dsId and not (directConnId and directService): - return ToolResult(toolCallId="", toolName="searchDataSource", success=False, - error="Provide either dataSourceId OR connectionId+service") - try: - if dsId: - connectionId, service, basePath, _neutralize = await _resolveDataSource(dsId) - else: - connectionId, service, basePath = directConnId, directService, args.get("path", "/") - from modules.connectors.connectorResolver import ConnectorResolver - resolver = ConnectorResolver( - services.getService("security"), - _buildResolverDb(), - ) - adapter = await resolver.resolveService(connectionId, service) - entries = await adapter.search(query, path=basePath) - if not entries: - return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data="No results found.") - lines = [f"- {e.name} (path: {e.path})" for e in entries] - result = "\n".join(lines) - if service in _MAIL_SERVICES: - result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." - return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data=result) - except Exception as e: - return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error=str(e)) - - async def _downloadFromDataSource(args: Dict[str, Any], context: Dict[str, Any]): - dsId = args.get("dataSourceId", "") - directConnId = args.get("connectionId", "") - directService = args.get("service", "") - filePath = args.get("filePath", "") - fileName = args.get("fileName", "") - if not filePath: - return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="filePath is required") - if not dsId and not (directConnId and directService): - return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, - error="Provide either dataSourceId OR connectionId+service") - try: - from modules.connectors.connectorResolver import ConnectorResolver - from modules.connectors.connectorProviderBase import DownloadResult as _DR - _sourceNeutralize = False - if dsId: - connectionId, service, basePath, _sourceNeutralize = await _resolveDataSource(dsId) - else: - connectionId, service, basePath = directConnId, directService, "/" - fullPath = filePath if filePath.startswith("/") else f"{basePath.rstrip('/')}/{filePath}" - resolver = ConnectorResolver( - services.getService("security"), - _buildResolverDb(), - ) - adapter = await resolver.resolveService(connectionId, service) - result = await adapter.download(fullPath) - - if isinstance(result, _DR): - fileBytes = result.data - fileName = result.fileName or fileName - else: - fileBytes = result - - if not fileBytes: - return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="Download returned empty") - - if not fileName or "." not in fileName: - pathSegment = fullPath.split("/")[-1] or "downloaded_file" - fileName = fileName or pathSegment - if "." not in fileName: - try: - entries = await adapter.browse(basePath) - for entry in entries: - if getattr(entry, "path", "") == filePath or getattr(entry, "path", "").endswith(filePath): - if "." in entry.name: - fileName = entry.name - break - except Exception: - pass - if "." not in fileName: - if fileBytes[:4] == b"%PDF": - fileName = f"{fileName}.pdf" - elif fileBytes[:2] == b"PK": - fileName = f"{fileName}.zip" - chatService = services.chat - fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(fileBytes, fileName) - fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") - if fiId: - chatService.interfaceDbComponent.updateFile(fileItem.id, {"featureInstanceId": fiId}) - if _sourceNeutralize: - chatService.interfaceDbComponent.updateFile(fileItem.id, {"neutralize": True}) - tempFolderId = _getOrCreateTempFolder(chatService) - if tempFolderId: - chatService.interfaceDbComponent.updateFile(fileItem.id, {"folderId": tempFolderId}) - ext = fileName.rsplit(".", 1)[-1].lower() if "." in fileName else "" - hint = "Use readFile to read the text content." if ext in ("doc", "docx", "txt", "csv", "json", "xml", "html", "md", "rtf", "odt", "xls", "xlsx", "pptx", "pdf", "eml", "msg") else "Use readFile to access the content." - return ToolResult( - toolCallId="", toolName="downloadFromDataSource", success=True, - data=f"Downloaded '{fileName}' ({len(fileBytes)} bytes) → local file id: {fileItem.id}. {hint}" - ) - except Exception as e: - return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error=str(e)) - - registry.register( - "browseDataSource", _browseDataSource, - description=( - "Browse files and folders in a data source. Accepts either:\n" - "- dataSourceId (for attached data sources shown in the prompt), OR\n" - "- connectionId + service (for direct connection access via listConnections)." - ), - parameters={ - "type": "object", - "properties": { - "dataSourceId": {"type": "string", "description": "DataSource ID (from attached data sources)"}, - "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, - "service": {"type": "string", "description": "Service name (alternative to dataSourceId, e.g. sharepoint, onedrive)"}, - "path": {"type": "string", "description": "Root path (used with connectionId+service)"}, - "subPath": {"type": "string", "description": "Sub-path within the data source to browse"}, - "filter": {"type": "string", "description": "Filter pattern (e.g. '*.pdf')"}, - }, - }, - readOnly=True, - ) - - registry.register( - "searchDataSource", _searchDataSource, - description=( - "Search for files within a data source. Accepts either dataSourceId OR connectionId+service." - ), - parameters={ - "type": "object", - "properties": { - "dataSourceId": {"type": "string", "description": "DataSource ID"}, - "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, - "service": {"type": "string", "description": "Service name (alternative to dataSourceId)"}, - "path": {"type": "string", "description": "Scope path (used with connectionId+service)"}, - "query": {"type": "string", "description": "Search query"}, - }, - "required": ["query"], - }, - readOnly=True, - ) - - registry.register( - "downloadFromDataSource", _downloadFromDataSource, - description=( - "Download a file or email from a data source into local storage. Returns a local file ID " - "to read with readFile. Accepts either dataSourceId OR connectionId+service. " - "For email sources (Outlook, Gmail), browse/search only return subjects -- use this to get full content." - ), - parameters={ - "type": "object", - "properties": { - "dataSourceId": {"type": "string", "description": "DataSource ID"}, - "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, - "service": {"type": "string", "description": "Service name (alternative to dataSourceId)"}, - "filePath": {"type": "string", "description": "Path of the file to download (from browseDataSource results)"}, - "fileName": {"type": "string", "description": "File name with extension (e.g. 'report.pdf')"}, - }, - "required": ["dataSourceId", "filePath"], - }, - readOnly=False, - ) - - # ---- Document tools (Smart Documents / Container Handling) ---- - - async def _browseContainer(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="browseContainer", success=False, error="fileId is required") - try: - knowledgeService = services.getService("knowledge") - index = knowledgeService.getFileContentIndex(fileId) - if not index: - return ToolResult(toolCallId="", toolName="browseContainer", success=True, data="No content index available for this file. It may not have been indexed yet.") - structure = index.get("structure", {}) if isinstance(index, dict) else {} - objectSummary = index.get("objectSummary", []) if isinstance(index, dict) else [] - totalObjects = index.get("totalObjects", 0) if isinstance(index, dict) else 0 - - result = f"File: {index.get('fileName', '?')} ({index.get('mimeType', '?')})\n" - result += f"Total content objects: {totalObjects}\n" - - sections = structure.get("sections", []) - if sections: - result += "\nSections:\n" - for s in sections: - result += f" [{s.get('id', '?')}] {s.get('title', 'Untitled')} (pages {s.get('startPage', '?')}-{s.get('endPage', '?')})\n" - - if structure.get("pageMap"): - pages = len(structure["pageMap"]) - result += f"\nPages: {pages}\n" - imgCount = structure.get("imageCount", 0) - tableCount = structure.get("tableCount", 0) - if imgCount: - result += f"Images: {imgCount}\n" - if tableCount: - result += f"Tables: {tableCount}\n" - - if structure.get("sheetMap"): - result += "\nSheets:\n" - for s in structure["sheetMap"]: - result += f" {s.get('sheetName', '?')} ({s.get('rows', '?')} rows x {s.get('columns', '?')} cols)\n" - - if structure.get("slideMap"): - result += "\nSlides:\n" - for s in structure["slideMap"]: - result += f" Slide {s.get('slideIndex', 0) + 1}: {s.get('title', '(no title)')}\n" - - return ToolResult(toolCallId="", toolName="browseContainer", success=True, data=result) - except Exception as e: - return ToolResult(toolCallId="", toolName="browseContainer", success=False, error=str(e)) - - async def _readContentObjects(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error="fileId is required") - try: - knowledgeService = services.getService("knowledge") - filterDict = {} - if args.get("pageIndex") is not None: - filterDict["pageIndex"] = args["pageIndex"] - if args.get("contentType"): - filterDict["contentType"] = args["contentType"] - if args.get("sectionId"): - filterDict["sectionId"] = args["sectionId"] - - objects = await knowledgeService.readContentObjects(fileId, filterDict) - if not objects: - return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data="No content objects found with the given filter.") - - result = f"Found {len(objects)} content objects:\n\n" - for obj in objects[:20]: - data = obj.get("data", "") - cType = obj.get("contentType", "?") - ref = obj.get("contextRef", {}) - location = ref.get("location", "") if isinstance(ref, dict) else "" - preview = data[:300] if cType == "text" else f"[{cType} data, {len(data)} chars]" - result += f"[{cType}] {location}: {preview}\n\n" - - if len(objects) > 20: - result += f"... and {len(objects) - 20} more objects" - - return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data=result) - except Exception as e: - return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error=str(e)) - - async def _extractContainerItem(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - containerPath = args.get("containerPath", "") - if not fileId or not containerPath: - return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error="fileId and containerPath are required") - try: - knowledgeService = services.getService("knowledge") - result = await knowledgeService.extractContainerItem(fileId, containerPath) - if result: - return ToolResult(toolCallId="", toolName="extractContainerItem", success=True, data=str(result)) - return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error=f"Item '{containerPath}' not found in container index for file {fileId}. On-demand extraction is not yet implemented.") - except Exception as e: - return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error=str(e)) - - async def _summarizeContent(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error="fileId is required") - try: - knowledgeService = services.getService("knowledge") - filterDict = {} - if args.get("sectionId"): - filterDict["sectionId"] = args["sectionId"] - if args.get("pageIndex") is not None: - filterDict["pageIndex"] = args["pageIndex"] - if args.get("contentType"): - filterDict["contentType"] = args["contentType"] - - objects = await knowledgeService.readContentObjects(fileId, filterDict) - if not objects: - return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data="No content found to summarize.") - - textParts = [obj.get("data", "") for obj in objects if obj.get("contentType") != "image"] - combinedText = "\n\n".join(textParts)[:6000] - - aiService = services.ai - from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum - summaryRequest = AiCallRequest( - prompt=f"Summarize the following content concisely:\n\n{combinedText}", - options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE), - ) - response = await aiService.callAi(summaryRequest) - return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data=response.content) - except Exception as e: - return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error=str(e)) - - registry.register( - "browseContainer", _browseContainer, - description="Browse the structural index of a document (pages, sections, sheets, slides). Use before readContentObjects for targeted reading.", - parameters={ - "type": "object", - "properties": {"fileId": {"type": "string", "description": "The file ID to browse"}}, - "required": ["fileId"], - }, - readOnly=True, - ) - - registry.register( - "readContentObjects", _readContentObjects, - description="Read extracted content objects from a file, optionally filtered by page, section, or type. Use browseContainer first to see the structure.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID"}, - "pageIndex": {"type": "integer", "description": "Filter by page index"}, - "sectionId": {"type": "string", "description": "Filter by section ID"}, - "contentType": {"type": "string", "description": "Filter by content type (text, image, etc.)"}, - }, - "required": ["fileId"], - }, - readOnly=True, - ) - - registry.register( - "extractContainerItem", _extractContainerItem, - description="Extract a specific item from a container file (ZIP, nested file). Use browseContainer to see available items.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The container file ID"}, - "containerPath": {"type": "string", "description": "Path within the container"}, - }, - "required": ["fileId", "containerPath"], - }, - readOnly=True, - ) - - registry.register( - "summarizeContent", _summarizeContent, - description="Generate an AI-powered summary of a file's content. Optionally filter by section, page, or content type.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID"}, - "sectionId": {"type": "string", "description": "Optional: summarize only this section"}, - "pageIndex": {"type": "integer", "description": "Optional: summarize only this page"}, - "contentType": {"type": "string", "description": "Optional: filter by content type"}, - }, - "required": ["fileId"], - }, - readOnly=True, - ) - - # ---- Vision tool ---- - - async def _describeImage(args: Dict[str, Any], context: Dict[str, Any]): - """Analyse an image using AI vision. Uses Knowledge Store chunks produced by Extractors.""" - fileId = args.get("fileId", "") - prompt = args.get("prompt", "Describe this image in detail. Extract all visible text, tables, and data.") - pageIndex = args.get("pageIndex") - - if not fileId: - return ToolResult(toolCallId="", toolName="describeImage", success=False, error="fileId is required") - - try: - import base64 as _b64 - - imageData = None - mimeType = "image/png" - - knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None - - # 1) Knowledge Store: image chunks already produced by PdfExtractor / ImageExtractor - if knowledgeService: - chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) - imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] - if pageIndex is not None: - imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] - if imageChunks: - imageData = imageChunks[0].get("data", "") - chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType") - if chunkMime: - mimeType = chunkMime - - # 2) File not yet indexed -> trigger extraction via ExtractionService, then retry - if not imageData and knowledgeService and not knowledgeService.isFileIndexed(fileId): - try: - chatService = services.chat - fileInfo = chatService.getFileInfo(fileId) - fileContent = chatService.getFileContent(fileId) - if fileContent and fileInfo: - rawData = fileContent.get("data", "") - if isinstance(rawData, str) and len(rawData) > 100: - rawBytes = _b64.b64decode(rawData) - elif isinstance(rawData, bytes): - rawBytes = rawData - else: - rawBytes = None - - if rawBytes: - from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry - from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction - from modules.datamodels.datamodelExtraction import ExtractionOptions - - fileMime = fileInfo.get("mimeType", "application/octet-stream") - fileName = fileInfo.get("fileName", fileId) - extracted = runExtraction( - ExtractorRegistry(), None, - rawBytes, fileName, fileMime, ExtractionOptions(), - ) - - contentObjects = [] - for part in extracted.parts: - tg = (part.typeGroup or "").lower() - ct = "image" if tg == "image" else "text" - if not part.data or not part.data.strip(): - continue - contentObjects.append({ - "contentObjectId": part.id, - "contentType": ct, - "data": part.data, - "contextRef": {"containerPath": fileName, "location": part.label, **(part.metadata or {})}, - }) - - if contentObjects: - _diFiId, _diMId = _resolveFileScope(fileId, context) - await knowledgeService.indexFile( - fileId=fileId, fileName=fileName, mimeType=fileMime, - userId=context.get("userId", ""), contentObjects=contentObjects, - featureInstanceId=_diFiId, - mandateId=_diMId, - ) - - chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) - imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] - if pageIndex is not None: - imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] - if imageChunks: - imageData = imageChunks[0].get("data", "") - except Exception as extractErr: - logger.warning(f"describeImage: on-demand extraction failed: {extractErr}") - - # 3) Direct image file (not a container) - use raw file data - if not imageData: - chatService = services.chat - fileContent = chatService.getFileContent(fileId) - if fileContent: - fileMimeType = fileContent.get("mimeType", "") - if fileMimeType.startswith("image/"): - imageData = fileContent.get("data", "") - mimeType = fileMimeType - - if not imageData: - chatService = services.chat - fileInfo = chatService.getFileInfo(fileId) if hasattr(chatService, "getFileInfo") else None - fileName = fileInfo.get("fileName", fileId) if fileInfo else fileId - fileMime = fileInfo.get("mimeType", "unknown") if fileInfo else "unknown" - return ToolResult(toolCallId="", toolName="describeImage", success=False, - error=f"No image data found in '{fileName}' (type: {fileMime}). " - f"This file likely contains text, not images. Use readFile(fileId=\"{fileId}\") to access its text content.") - - try: - rawHead = _b64.b64decode(imageData[:32]) - if rawHead[:3] == b"\xff\xd8\xff": - mimeType = "image/jpeg" - elif rawHead[:8] == b"\x89PNG\r\n\x1a\n": - mimeType = "image/png" - elif rawHead[:4] == b"GIF8": - mimeType = "image/gif" - elif rawHead[:4] == b"RIFF" and rawHead[8:12] == b"WEBP": - mimeType = "image/webp" - except Exception: - pass - dataUrl = f"data:{mimeType};base64,{imageData}" - from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum as OTE - - _opType = OTE.IMAGE_ANALYSE - try: - from modules.datamodels.datamodelFiles import FileItem as _FileItemModel - from modules.interfaces.interfaceDbManagement import ComponentObjects as _CO - _fRow = _CO().db._loadRecord(_FileItemModel, fileId) - if _fRow: - _fGet = (lambda k, d=None: _fRow.get(k, d)) if isinstance(_fRow, dict) else (lambda k, d=None: getattr(_fRow, k, d)) - if bool(_fGet("neutralize", False)): - _opType = OTE.NEUTRALIZATION_IMAGE - logger.info(f"describeImage: file {fileId} has neutralize=True, using NEUTRALIZATION_IMAGE (internal models only)") - except Exception: - pass - - visionRequest = AiCallRequest( - prompt=prompt, - options=AiCallOptions(operationType=_opType), - messages=[{"role": "user", "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": dataUrl}}, - ]}], - ) - visionResponse = await services.ai.callAi(visionRequest) - - if visionResponse.errorCount > 0: - return ToolResult(toolCallId="", toolName="describeImage", success=False, error=visionResponse.content) - return ToolResult(toolCallId="", toolName="describeImage", success=True, data=visionResponse.content) - - except Exception as e: - return ToolResult(toolCallId="", toolName="describeImage", success=False, error=str(e)) - - registry.register( - "describeImage", _describeImage, - description="Analyze an image using AI vision. Works with image files and images extracted from PDFs/DOCX/PPTX. Use for OCR, data extraction, and visual analysis.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "The file ID containing the image or document with images"}, - "prompt": {"type": "string", "description": "What to look for in the image (default: describe everything)"}, - "pageIndex": {"type": "integer", "description": "Filter images by page index (0-based, for multi-page documents)"}, - }, - "required": ["fileId"], - }, - readOnly=True, - ) - - # ---- Document rendering tool ---- - - def _markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]: - """Convert markdown content to the standard document JSON format expected by renderers.""" - import re as _re - - sections = [] - order = 0 - lines = markdown.split("\n") - i = 0 - - def _nextId(): - nonlocal order - order += 1 - return f"s_{order}" - - while i < len(lines): - line = lines[i] - - # --- Headings --- - headingMatch = _re.match(r'^(#{1,6})\s+(.+)', line) - if headingMatch: - level = len(headingMatch.group(1)) - text = headingMatch.group(2).strip() - sections.append({ - "id": _nextId(), "content_type": "heading", "order": order, - "elements": [{"content": {"text": text, "level": level}}], - }) - i += 1 - continue - - # --- Fenced code blocks --- - codeMatch = _re.match(r'^```(\w*)', line) - if codeMatch: - lang = codeMatch.group(1) or "text" - codeLines = [] - i += 1 - while i < len(lines) and not lines[i].startswith("```"): - codeLines.append(lines[i]) - i += 1 - i += 1 - sections.append({ - "id": _nextId(), "content_type": "code_block", "order": order, - "elements": [{"content": {"code": "\n".join(codeLines), "language": lang}}], - }) - continue - - # --- Tables --- - tableMatch = _re.match(r'^\|(.+)\|$', line) - if tableMatch and (i + 1) < len(lines) and _re.match(r'^\|[\s\-:|]+\|$', lines[i + 1]): - headerCells = [c.strip() for c in tableMatch.group(1).split("|")] - i += 2 - rows = [] - while i < len(lines) and _re.match(r'^\|(.+)\|$', lines[i]): - rowCells = [c.strip() for c in lines[i][1:-1].split("|")] - rows.append(rowCells) - i += 1 - sections.append({ - "id": _nextId(), "content_type": "table", "order": order, - "elements": [{"content": {"headers": headerCells, "rows": rows}}], - }) - continue - - # --- Bullet / numbered lists --- - listMatch = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', line) - if listMatch: - isNumbered = bool(_re.match(r'\d+[.)]', listMatch.group(2))) - items = [] - while i < len(lines) and _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]): - m = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]) - items.append({"text": m.group(3).strip()}) - i += 1 - sections.append({ - "id": _nextId(), "content_type": "bullet_list", "order": order, - "elements": [{"content": {"items": items, "list_type": "numbered" if isNumbered else "bullet"}}], - }) - continue - - # --- Empty lines (skip) --- - if not line.strip(): - i += 1 - continue - - # --- Images: ![alt](file:fileId) or ![alt](url) --- - imgMatch = _re.match(r'^!\[([^\]]*)\]\(([^)]+)\)', line) - if imgMatch: - altText = imgMatch.group(1).strip() or "Image" - src = imgMatch.group(2).strip() - fileId = "" - if src.startswith("file:"): - fileId = src[5:] - sections.append({ - "id": _nextId(), "content_type": "image", "order": order, - "elements": [{ - "content": { - "altText": altText, - "base64Data": "", - "_fileRef": fileId, - "_srcUrl": src if not fileId else "", - } - }], - }) - i += 1 - continue - - # --- Paragraph (collect consecutive non-empty lines) --- - paraLines = [] - while i < len(lines) and lines[i].strip() and not _re.match(r'^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )', lines[i]): - paraLines.append(lines[i]) - i += 1 - if paraLines: - sections.append({ - "id": _nextId(), "content_type": "paragraph", "order": order, - "elements": [{"content": {"text": " ".join(paraLines)}}], - }) - continue - - i += 1 - - if not sections: - sections.append({ - "id": _nextId(), "content_type": "paragraph", "order": order, - "elements": [{"content": {"text": markdown.strip() or "(empty)"}}], - }) - - return { - "metadata": { - "split_strategy": "single_document", - "source_documents": [], - "extraction_method": "agent_rendering", - "title": title, - "language": language, - }, - "documents": [{ - "id": "doc_1", - "title": title, - "sections": sections, - }], - } - - async def _renderDocument(args: Dict[str, Any], context: Dict[str, Any]): - """Render agent-produced markdown content into any document format via the RendererRegistry.""" - import re as _re - sourceFileId = (args.get("sourceFileId") or "").strip() - content = args.get("content", "") - if not isinstance(content, str): - content = str(content) if content is not None else "" - outputFormat = args.get("outputFormat", "pdf") - title = args.get("title", "Document") - language = args.get("language", "de") - - if sourceFileId: - try: - dbMgmt = services.chat.interfaceDbComponent - fileRow = dbMgmt.getFile(sourceFileId) - if not fileRow: - return ToolResult( - toolCallId="", - toolName="renderDocument", - success=False, - error=f"sourceFileId not found: {sourceFileId}", - ) - rawBytes = dbMgmt.getFileData(sourceFileId) - if not rawBytes: - return ToolResult( - toolCallId="", - toolName="renderDocument", - success=False, - error=f"sourceFileId has no data: {sourceFileId}", - ) - try: - content = rawBytes.decode("utf-8") - except UnicodeDecodeError: - content = rawBytes.decode("latin-1", errors="replace") - except Exception as e: - return ToolResult( - toolCallId="", - toolName="renderDocument", - success=False, - error=f"Could not read sourceFileId: {e}", - ) - - if not (content or "").strip(): - return ToolResult( - toolCallId="", - toolName="renderDocument", - success=False, - error=( - "Provide non-empty `content` (markdown) or `sourceFileId` (id of a .md/.txt from writeFile). " - "For long documents use writeFile create+append, then renderDocument(sourceFileId=...)." - ), - ) - - modelMaxTokens = context.get("modelMaxOutputTokens", 0) - _inlineCharLimit = int(modelMaxTokens * 3 * 0.5) if modelMaxTokens > 0 else 6000 - _inlineCharLimit = max(_inlineCharLimit, 3000) - - if not sourceFileId and len(content) > _inlineCharLimit: - return ToolResult( - toolCallId="", - toolName="renderDocument", - success=False, - error=( - f"Inline `content` is {len(content)} chars — over the {_inlineCharLimit} char limit " - f"(derived from model output budget of {modelMaxTokens} tokens). " - "Large documents must use the file path:\n" - "1. writeFile(mode='create', name='draft.md', content=)\n" - "2. writeFile(mode='append', fileId=, content=) — repeat as needed\n" - "3. renderDocument(sourceFileId=, outputFormat='pdf', title='...')\n" - "This avoids output truncation entirely." - ), - ) - - try: - structuredContent = _markdownToDocumentJson(content, title, language) - - # Resolve image file references (file:fileId) to base64 data from Knowledge Store - knowledgeService = None - try: - knowledgeService = services.getService("knowledge") - except Exception: - pass - resolvedImages = 0 - for doc in structuredContent.get("documents", []): - for section in doc.get("sections", []): - if section.get("content_type") != "image": - continue - for element in section.get("elements", []): - contentObj = element.get("content", {}) - fileRef = contentObj.get("_fileRef", "") - if not fileRef or contentObj.get("base64Data"): - continue - if knowledgeService: - chunks = knowledgeService._knowledgeDb.getContentChunks(fileRef) - imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] - if imageChunks: - contentObj["base64Data"] = imageChunks[0].get("data", "") - chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType", "image/png") - contentObj["mimeType"] = chunkMime - resolvedImages += 1 - if not contentObj.get("base64Data"): - try: - rawBytes = services.chat.getFileData(fileRef) - if rawBytes: - import base64 as _b64 - contentObj["base64Data"] = _b64.b64encode(rawBytes).decode("ascii") - contentObj["mimeType"] = "image/png" - resolvedImages += 1 - except Exception: - pass - contentObj.pop("_fileRef", None) - contentObj.pop("_srcUrl", None) - - sectionCount = len(structuredContent.get("documents", [{}])[0].get("sections", [])) - logger.info(f"renderDocument: parsed {sectionCount} sections from markdown ({len(content)} chars), resolved {resolvedImages} image(s), format={outputFormat}") - - generationService = services.getService("generation") - documents = await generationService.renderReport( - extractedContent=structuredContent, - outputFormat=outputFormat, - language=language, - title=title, - userPrompt=content, - ) - - if not documents: - return ToolResult(toolCallId="", toolName="renderDocument", success=False, error="Rendering produced no output") - - savedFiles = [] - sideEvents = [] - chatService = services.chat - - sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "document" - - for doc in documents: - docData = doc.documentData if hasattr(doc, "documentData") else b"" - docName = doc.filename if hasattr(doc, "filename") else f"{sanitizedTitle}.{outputFormat}" - docMime = doc.mimeType if hasattr(doc, "mimeType") else "application/octet-stream" - - if not docName.lower().endswith(f".{outputFormat}"): - docName = f"{sanitizedTitle}.{outputFormat}" - - fileItem = None - if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): - fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) - else: - fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) - - if fileItem: - fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") - fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") - if fiId: - chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) - tempFolderId = _getOrCreateTempFolder(chatService) - if tempFolderId: - chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) - savedFiles.append(f"- {docName} (id: {fid})") - sideEvents.append({ - "type": "fileCreated", - "data": { - "fileId": fid, - "fileName": docName, - "mimeType": docMime, - "fileSize": len(docData), - }, - }) - - result = f"Rendered {len(documents)} document(s):\n" + "\n".join(savedFiles) - return ToolResult(toolCallId="", toolName="renderDocument", success=True, data=result, sideEvents=sideEvents) - - except Exception as e: - logger.error(f"renderDocument failed: {e}") - return ToolResult(toolCallId="", toolName="renderDocument", success=False, error=str(e)) - - registry.register( - "renderDocument", _renderDocument, - description=( - "Render markdown into a document file (PDF, DOCX, XLSX, PPTX, CSV, HTML, MD, JSON, TXT). " - "For long documents: write markdown with writeFile (mode=create then append chunks), then call this tool with " - "`sourceFileId` only (tiny JSON — avoids model output truncation). For short docs you may pass `content` inline. " - "Images: ![alt text](file:fileId) in the markdown." - ), - parameters={ - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "Full markdown inline. Prefer `sourceFileId` when the document is large (many KB).", - }, - "sourceFileId": { - "type": "string", - "description": "Chat file id of markdown saved via writeFile (create+append). Use this instead of `content` for long PDFs.", - }, - "outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"}, - "title": {"type": "string", "description": "Document title", "default": "Document"}, - "language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"}, - }, - }, - readOnly=False, - ) - - # ── textToSpeech tool ────────────────────────────────────────────── - def _stripMarkdownForTts(text: str) -> str: - """Strip markdown formatting so TTS reads clean speech text.""" - import re as _re - t = text - t = _re.sub(r'\*\*(.+?)\*\*', r'\1', t) - t = _re.sub(r'\*(.+?)\*', r'\1', t) - t = _re.sub(r'__(.+?)__', r'\1', t) - t = _re.sub(r'_(.+?)_', r'\1', t) - t = _re.sub(r'`[^`]+`', lambda m: m.group(0)[1:-1], t) - t = _re.sub(r'^#{1,6}\s*', '', t, flags=_re.MULTILINE) - t = _re.sub(r'^\s*[-*+]\s+', '', t, flags=_re.MULTILINE) - t = _re.sub(r'^\s*\d+\.\s+', '', t, flags=_re.MULTILINE) - t = _re.sub(r'\[(.+?)\]\(.+?\)', r'\1', t) - t = _re.sub(r'!\[.*?\]\(.*?\)', '', t) - t = _re.sub(r'\n{3,}', '\n\n', t) - return t.strip() - - async def _textToSpeech(args: Dict[str, Any], context: Dict[str, Any]): - """Convert text to speech using Google Cloud TTS, deliver audio via SSE.""" - import base64 as _b64 - text = args.get("text", "") - language = args.get("language", "auto") - voiceName = args.get("voiceName") - - if not text: - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is required") - - cleanText = _stripMarkdownForTts(text) - if not cleanText: - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is empty after stripping markdown") - - try: - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - mandateId = context.get("mandateId", "") - voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) - - _ISO_TO_BCP47 = { - "de": "de-DE", "en": "en-US", "fr": "fr-FR", "it": "it-IT", - "es": "es-ES", "pt": "pt-BR", "nl": "nl-NL", "pl": "pl-PL", - "ru": "ru-RU", "ja": "ja-JP", "zh": "zh-CN", "ko": "ko-KR", - "ar": "ar-XA", "hi": "hi-IN", "tr": "tr-TR", "sv": "sv-SE", - } - - if language == "auto": - try: - snippet = cleanText[:500] - detectResult = await voiceInterface.detectLanguage(snippet) - if detectResult and detectResult.get("success"): - detected = detectResult.get("language", "de") - language = _ISO_TO_BCP47.get(detected, detected) - if "-" not in language: - language = _ISO_TO_BCP47.get(language, f"{language}-{language.upper()}") - logger.info(f"textToSpeech: auto-detected language '{detected}' -> '{language}'") - else: - language = "de-DE" - except Exception as detectErr: - logger.warning(f"textToSpeech: language detection failed: {detectErr}, defaulting to de-DE") - language = "de-DE" - - if not voiceName: - try: - from modules.datamodels.datamodelUam import UserVoicePreferences - from modules.interfaces.interfaceDbApp import getRootInterface - userId = context.get("userId", "") - if userId: - rootIf = getRootInterface() - prefRecords = rootIf.db.getRecordset( - UserVoicePreferences, - recordFilter={"userId": userId} - ) - if prefRecords: - allPrefs = [ - r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else r - for r in prefRecords - ] - _mid = str(mandateId or "").strip() - scopedPref = next((p for p in allPrefs if str(p.get("mandateId") or "").strip() == _mid), None) - globalPref = next((p for p in allPrefs if not str(p.get("mandateId") or "").strip()), None) - - def _resolveVoiceFromMap(prefDict, lang): - vm = (prefDict or {}).get("ttsVoiceMap", {}) or {} - if not isinstance(vm, dict) or not vm: - return None - baseLang = lang.split("-")[0].lower() if isinstance(lang, str) and lang else "" - langNorm = str(lang or "").strip() - if langNorm in vm: - entry = vm[langNorm] - return entry.get("voiceName") if isinstance(entry, dict) else entry - if baseLang and baseLang in vm: - entry = vm[baseLang] - return entry.get("voiceName") if isinstance(entry, dict) else entry - if baseLang: - for mk, mv in vm.items(): - mkn = str(mk).lower() - if mkn == baseLang or mkn.startswith(f"{baseLang}-"): - return mv.get("voiceName") if isinstance(mv, dict) else mv - return None - - voiceName = ( - _resolveVoiceFromMap(scopedPref, language) - or _resolveVoiceFromMap(globalPref, language) - or _resolveVoiceFromMap(allPrefs[0], language) - ) - if not voiceName: - for candidate in [globalPref, scopedPref, allPrefs[0]]: - if candidate and candidate.get("ttsVoice") and candidate.get("ttsLanguage") == language: - voiceName = candidate["ttsVoice"] - break - if voiceName: - logger.info(f"textToSpeech: using configured voice '{voiceName}' for language '{language}'") - except Exception as prefErr: - logger.debug(f"textToSpeech: could not load voice preferences: {prefErr}") - - ttsResult = await voiceInterface.textToSpeech( - text=cleanText, - languageCode=language, - voiceName=voiceName, - ) - - if not ttsResult or not ttsResult.get("success"): - errMsg = ttsResult.get("error", "TTS call failed") if ttsResult else "TTS returned None" - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=errMsg) - - audioContent = ttsResult.get("audioContent", "") - if not audioContent: - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="TTS returned no audio") - - if isinstance(audioContent, bytes): - audioB64 = _b64.b64encode(audioContent).decode("ascii") - elif isinstance(audioContent, str): - audioB64 = audioContent - else: - audioB64 = str(audioContent) - - audioFormat = ttsResult.get("audioFormat", "mp3") - charCount = len(cleanText) - usedVoice = voiceName or "default" - logger.info(f"textToSpeech: generated {audioFormat} audio for {charCount} chars, language={language}, voice={usedVoice}") - - return ToolResult( - toolCallId="", toolName="textToSpeech", success=True, - data=f"Audio generated ({charCount} characters, language={language}, voice={usedVoice}). Playing in chat.", - sideEvents=[{ - "type": "voiceResponse", - "data": { - "audio": audioB64, - "format": audioFormat, - "language": language, - "charCount": charCount, - }, - }], - ) - - except ImportError: - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, - error="Voice interface not available (missing dependency)") - except Exception as e: - logger.error(f"textToSpeech failed: {e}") - return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=str(e)) - - registry.register( - "textToSpeech", _textToSpeech, - description=( - "Convert text to speech audio. The audio is played directly in the chat. " - "Use this when the user asks you to read something aloud, narrate, or speak. " - "Language is auto-detected from the text content. You do NOT need to specify a language." - ), - parameters={ - "type": "object", - "properties": { - "text": {"type": "string", "description": "The text to convert to speech. Can include markdown (will be stripped automatically)."}, - "language": {"type": "string", "description": "BCP-47 language code (e.g. de-DE, en-US) or 'auto' for automatic detection", "default": "auto"}, - "voiceName": {"type": "string", "description": "Optional specific voice name. If omitted, uses the configured voice for the detected language."}, - }, - "required": ["text"], - }, - readOnly=False, - ) - - # ── generateImage tool ───────────────────────────────────────────── - - async def _generateImage(args: Dict[str, Any], context: Dict[str, Any]): - """Generate an image from a text prompt using AI (DALL-E).""" - import re as _re - - prompt = (args.get("prompt") or "").strip() - style = (args.get("style") or "").strip() or None - title = (args.get("title") or "").strip() or "Generated Image" - - if not prompt: - return ToolResult(toolCallId="", toolName="generateImage", success=False, error="prompt is required") - - try: - from modules.serviceCenter.services.serviceGeneration.paths.imagePath import ImageGenerationPath - - imagePath = ImageGenerationPath(services) - aiResponse = await imagePath.generateImages( - userPrompt=prompt, - count=1, - style=style, - format="png", - title=title, - ) - - if not aiResponse.documents: - return ToolResult(toolCallId="", toolName="generateImage", success=False, error="Image generation returned no image data") - - sideEvents = [] - savedFiles = [] - chatService = services.chat - sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "generated_image" - - for doc in aiResponse.documents: - docData = doc.documentData if hasattr(doc, "documentData") else b"" - docName = doc.documentName if hasattr(doc, "documentName") else f"{sanitizedTitle}.png" - docMime = doc.mimeType if hasattr(doc, "mimeType") else "image/png" - - if not docName.lower().endswith(".png"): - docName = f"{sanitizedTitle}.png" - - fileItem = None - if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): - fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) - else: - fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) - - if fileItem: - fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") - fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") - if fiId: - chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) - tempFolderId = _getOrCreateTempFolder(chatService) - if tempFolderId: - chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) - savedFiles.append(f"- {docName} (id: {fid})") - sideEvents.append({ - "type": "fileCreated", - "data": { - "fileId": fid, - "fileName": docName, - "mimeType": docMime, - "fileSize": len(docData), - }, - }) - - result = f"Generated {len(aiResponse.documents)} image(s):\n" + "\n".join(savedFiles) - return ToolResult(toolCallId="", toolName="generateImage", success=True, data=result, sideEvents=sideEvents) - - except Exception as e: - logger.error(f"generateImage failed: {e}") - return ToolResult(toolCallId="", toolName="generateImage", success=False, error=str(e)) - - registry.register( - "generateImage", _generateImage, - description=( - "Generate an image from a text description using AI (DALL-E). " - "The generated image is saved as a file in the workspace. " - "Use this when the user asks to create, generate, draw, or design an image, illustration, icon, logo, diagram, or any visual content. " - "Provide a detailed, descriptive prompt for best results." - ), - parameters={ - "type": "object", - "properties": { - "prompt": {"type": "string", "description": "Detailed description of the image to generate. Be specific about subject, composition, colors, style, and mood."}, - "style": {"type": "string", "description": "Optional style modifier (e.g. 'photorealistic', 'watercolor', 'digital art', 'minimalist', 'sketch')"}, - "title": {"type": "string", "description": "Title/filename for the generated image", "default": "Generated Image"}, - }, - "required": ["prompt"], - }, - readOnly=False, - ) - - # ── createChart tool ───────────────────────────────────────────────── - - async def _createChart(args: Dict[str, Any], context: Dict[str, Any]): - """Create a data chart as PNG image using matplotlib.""" - import re as _re - - chartType = (args.get("chartType") or "bar").strip().lower() - title = (args.get("title") or "Chart").strip() - labels = args.get("labels") or [] - datasets = args.get("datasets") or [] - xLabel = (args.get("xLabel") or "").strip() - yLabel = (args.get("yLabel") or "").strip() - width = min(max(args.get("width") or 10, 4), 20) - height = min(max(args.get("height") or 6, 3), 14) - colors = args.get("colors") or None - - if not datasets: - return ToolResult(toolCallId="", toolName="createChart", success=False, error="datasets is required (list of {label, values})") - - try: - import matplotlib - matplotlib.use("Agg") - import logging as _mpllog - _mpllog.getLogger("matplotlib").setLevel(_mpllog.WARNING) - import matplotlib.pyplot as plt - import io - - _DEFAULT_COLORS = [ - "#4285F4", "#EA4335", "#FBBC04", "#34A853", "#FF6D01", - "#46BDC6", "#7B61FF", "#F538A0", "#00ACC1", "#AB47BC", - ] - usedColors = colors if colors and len(colors) >= len(datasets) else _DEFAULT_COLORS - - fig, ax = plt.subplots(figsize=(width, height)) - fig.patch.set_facecolor("#FFFFFF") - ax.set_facecolor("#FAFAFA") - - if chartType in ("pie", "donut"): - values = datasets[0].get("values", []) if datasets else [] - explode = [0.02] * len(values) - wedges, texts, autotexts = ax.pie( - values, labels=labels, autopct="%1.1f%%", - colors=usedColors[:len(values)], explode=explode, - textprops={"fontsize": 9}, - ) - if chartType == "donut": - ax.add_artist(plt.Circle((0, 0), 0.55, fc="white")) - ax.set_title(title, fontsize=14, fontweight="bold", pad=16) - - else: - import numpy as _np - x = _np.arange(len(labels)) if labels else _np.arange(max(len(d.get("values", [])) for d in datasets)) - barWidth = 0.8 / max(len(datasets), 1) - - for i, ds in enumerate(datasets): - dsLabel = ds.get("label", f"Series {i+1}") - values = ds.get("values", []) - color = usedColors[i % len(usedColors)] - - if chartType == "bar": - offset = (i - len(datasets) / 2 + 0.5) * barWidth - ax.bar(x + offset, values, barWidth, label=dsLabel, color=color, edgecolor="white", linewidth=0.5) - elif chartType == "horizontalbar": - offset = (i - len(datasets) / 2 + 0.5) * barWidth - ax.barh(x + offset, values, barWidth, label=dsLabel, color=color, edgecolor="white", linewidth=0.5) - elif chartType == "line": - ax.plot(x[:len(values)], values, marker="o", markersize=5, label=dsLabel, color=color, linewidth=2) - elif chartType == "area": - ax.fill_between(x[:len(values)], values, alpha=0.3, color=color) - ax.plot(x[:len(values)], values, label=dsLabel, color=color, linewidth=2) - elif chartType == "scatter": - ax.scatter(x[:len(values)], values, label=dsLabel, color=color, s=50, edgecolors="white", linewidth=0.5) - else: - ax.bar(x, values, label=dsLabel, color=color) - - if labels: - if chartType == "horizontalbar": - ax.set_yticks(x) - ax.set_yticklabels(labels, fontsize=9) - else: - ax.set_xticks(x) - ax.set_xticklabels(labels, fontsize=9, rotation=45 if len(labels) > 6 else 0, ha="right" if len(labels) > 6 else "center") - - ax.set_title(title, fontsize=14, fontweight="bold", pad=12) - if xLabel: - ax.set_xlabel(xLabel, fontsize=10) - if yLabel: - ax.set_ylabel(yLabel, fontsize=10) - if len(datasets) > 1: - ax.legend(fontsize=9, framealpha=0.9) - ax.grid(axis="y", alpha=0.3, linestyle="--") - ax.spines["top"].set_visible(False) - ax.spines["right"].set_visible(False) - - plt.tight_layout() - buf = io.BytesIO() - fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") - plt.close(fig) - pngData = buf.getvalue() - - chatService = services.chat - sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "chart" - fileName = f"{sanitizedTitle}.png" - - if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): - fileItem = chatService.interfaceDbComponent.saveGeneratedFile(pngData, fileName, "image/png") - else: - fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(pngData, fileName) - - fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") if isinstance(fileItem, dict) else "?" - fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") - if fiId and fid != "?": - chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) - tempFolderId = _getOrCreateTempFolder(chatService) - if tempFolderId and fid != "?": - chatService.interfaceDbComponent.updateFile(fid, {"folderId": tempFolderId}) - - sideEvents = [{"type": "fileCreated", "data": { - "fileId": fid, "fileName": fileName, - "mimeType": "image/png", "fileSize": len(pngData), - }}] - return ToolResult( - toolCallId="", toolName="createChart", success=True, - data=f"Chart saved as '{fileName}' (id: {fid}, {len(pngData)} bytes). " - f"Embed in documents with: ![{title}](file:{fid})", - sideEvents=sideEvents, - ) - - except Exception as e: - logger.error(f"createChart failed: {e}", exc_info=True) - return ToolResult(toolCallId="", toolName="createChart", success=False, error=str(e)) - - registry.register( - "createChart", _createChart, - description=( - "Create a data chart/graph as a PNG image using matplotlib. " - "Supported types: bar, horizontalBar, line, area, scatter, pie, donut. " - "The chart is saved as a file in the workspace. " - "Use the returned fileId to embed in documents via renderDocument: ![title](file:fileId). " - "Provide structured data with labels and datasets." - ), - parameters={ - "type": "object", - "properties": { - "chartType": { - "type": "string", - "enum": ["bar", "horizontalBar", "line", "area", "scatter", "pie", "donut"], - "description": "Chart type (default: bar)", - }, - "title": {"type": "string", "description": "Chart title"}, - "labels": { - "type": "array", "items": {"type": "string"}, - "description": "X-axis labels / category names", - }, - "datasets": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": {"type": "string", "description": "Series name (legend)"}, - "values": {"type": "array", "items": {"type": "number"}, "description": "Data values"}, - }, - "required": ["values"], - }, - "description": "Data series to plot", - }, - "xLabel": {"type": "string", "description": "X-axis label"}, - "yLabel": {"type": "string", "description": "Y-axis label"}, - "colors": { - "type": "array", "items": {"type": "string"}, - "description": "Custom hex colors for series (e.g. ['#4285F4', '#EA4335'])", - }, - "width": {"type": "number", "description": "Figure width in inches (4-20, default 10)"}, - "height": {"type": "number", "description": "Figure height in inches (3-14, default 6)"}, - }, - "required": ["datasets"], - }, - readOnly=False, - ) - - # ── Phase 3: speechToText, detectLanguage, neutralizeData, executeCode ── - - async def _speechToText(args: Dict[str, Any], context: Dict[str, Any]): - fileId = args.get("fileId", "") - if not fileId: - return ToolResult(toolCallId="", toolName="speechToText", success=False, error="fileId is required") - try: - chatService = services.chat - audioData = chatService.interfaceDbComponent.getFileData(fileId) - if not audioData: - return ToolResult(toolCallId="", toolName="speechToText", success=False, error=f"No data found for file {fileId}") - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - mandateId = context.get("mandateId", "") - voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) - language = args.get("language", "de-DE") - result = await voiceInterface.speechToText(audioData, language=language) - if result and result.get("success"): - transcript = result.get("text", "") - confidence = result.get("confidence", 0) - return ToolResult( - toolCallId="", toolName="speechToText", success=True, - data=f"Transcript (confidence: {confidence:.0%}):\n{transcript}" - ) - return ToolResult(toolCallId="", toolName="speechToText", success=False, error=result.get("error", "Transcription failed")) - except Exception as e: - return ToolResult(toolCallId="", toolName="speechToText", success=False, error=str(e)) - - async def _detectLanguage(args: Dict[str, Any], context: Dict[str, Any]): - text = args.get("text", "") - if not text: - return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error="text is required") - try: - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - mandateId = context.get("mandateId", "") - voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) - result = await voiceInterface.detectLanguage(text) - if result and result.get("success"): - lang = result.get("language", "unknown") - return ToolResult(toolCallId="", toolName="detectLanguage", success=True, data=f"Detected language: {lang}") - return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error=result.get("error", "Detection failed")) - except Exception as e: - return ToolResult(toolCallId="", toolName="detectLanguage", success=False, error=str(e)) - - async def _neutralizeData(args: Dict[str, Any], context: Dict[str, Any]): - text = args.get("text", "") - fileId = args.get("fileId", "") - if not text and not fileId: - return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="text or fileId is required") - try: - neutralizationService = services.getService("neutralization") - if not neutralizationService: - return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="Neutralization service not available") - if not neutralizationService.interfaceDbComponent: - neutralizationService.interfaceDbComponent = services.chat.interfaceDbComponent - if text: - result = await neutralizationService.processTextAsync(text, fileId or None) - else: - result = neutralizationService.processFile(fileId) - if result: - neutralized = result.get("neutralized_text", "") or result.get("result", str(result)) - return ToolResult(toolCallId="", toolName="neutralizeData", success=True, data=neutralized) - return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error="Neutralization returned no result") - except Exception as e: - return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error=str(e)) - - async def _executeCode(args: Dict[str, Any], context: Dict[str, Any]): - code = args.get("code", "") - language = args.get("language", "python") - if not code: - return ToolResult(toolCallId="", toolName="executeCode", success=False, error="code is required") - if language != "python": - return ToolResult(toolCallId="", toolName="executeCode", success=False, error=f"Language '{language}' not supported. Only 'python' is available.") - try: - from modules.serviceCenter.services.serviceAgent.sandboxExecutor import executePython - result = await executePython(code) - if result.get("success"): - output = result.get("output", "(no output)") - return ToolResult(toolCallId="", toolName="executeCode", success=True, data=output) - error = result.get("error", "Execution failed") - tb = result.get("traceback", "") - return ToolResult(toolCallId="", toolName="executeCode", success=False, error=f"{error}\n{tb}" if tb else error) - except Exception as e: - return ToolResult(toolCallId="", toolName="executeCode", success=False, error=str(e)) - - registry.register( - "speechToText", _speechToText, - description="Transcribe an audio file to text using speech recognition. Returns the transcript with confidence score.", - parameters={ - "type": "object", - "properties": { - "fileId": {"type": "string", "description": "Audio file ID from the workspace"}, - "language": {"type": "string", "description": "BCP-47 language code (e.g. 'de-DE', 'en-US'). Default: 'de-DE'"}, - }, - "required": ["fileId"] - }, - readOnly=True - ) - - registry.register( - "detectLanguage", _detectLanguage, - description="Detect the language of a text snippet. Returns ISO 639-1 code (e.g. 'de', 'en').", - parameters={ - "type": "object", - "properties": { - "text": {"type": "string", "description": "Text to analyze"}, - }, - "required": ["text"] - }, - readOnly=True - ) - - registry.register( - "neutralizeData", _neutralizeData, - description="Anonymize text or file content by replacing personal data (names, addresses, etc.) with placeholders. Non-destructive -- returns the anonymized copy.", - parameters={ - "type": "object", - "properties": { - "text": {"type": "string", "description": "Text to anonymize"}, - "fileId": {"type": "string", "description": "File ID to anonymize (alternative to text)"}, - }, - }, - readOnly=True - ) - - registry.register( - "executeCode", _executeCode, - description=( - "Execute Python code in a sandboxed environment for calculations and data analysis. " - "Available modules: math, statistics, json, csv, re, datetime, collections, itertools, functools, decimal, fractions, random. " - "No file system, network, or OS access. Max 30s execution time. " - "Use print() to produce output." - ), - parameters={ - "type": "object", - "properties": { - "code": {"type": "string", "description": "Python code to execute"}, - "language": {"type": "string", "description": "Programming language (only 'python' supported)", "default": "python"}, - }, - "required": ["code"] - }, - readOnly=True - ) - - # ---- Feature Data Sub-Agent tool ---- - - async def _queryFeatureInstance(args: Dict[str, Any], context: Dict[str, Any]): - """Delegate a question to the Feature Data Sub-Agent.""" - featureInstanceId = args.get("featureInstanceId", "") - question = args.get("question", "") - if not featureInstanceId or not question: - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error="featureInstanceId and question are required", - ) - try: - from modules.serviceCenter.services.serviceAgent.featureDataAgent import runFeatureDataAgent - from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource - from modules.interfaces.interfaceDbApp import getRootInterface - - rootIf = getRootInterface() - instance = rootIf.getFeatureInstance(featureInstanceId) - if not instance: - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error=f"Feature instance {featureInstanceId} not found", - ) - - featureCode = instance.featureCode - mandateId = instance.mandateId or "" - instanceLabel = instance.label or "" - userId = context.get("userId", "") - workspaceInstanceId = context.get("featureInstanceId", "") - - rootDbConn = rootIf.db if hasattr(rootIf, "db") else None - if rootDbConn is None: - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error="No database connector available", - ) - - featureDataSources = rootDbConn.getRecordset( - FeatureDataSource, - recordFilter={"featureInstanceId": featureInstanceId, "workspaceInstanceId": workspaceInstanceId}, - ) - - _anySourceNeutralize = any( - bool(ds.get("neutralize", False) if isinstance(ds, dict) else getattr(ds, "neutralize", False)) - for ds in (featureDataSources or []) - ) - - from modules.security.rbacCatalog import getCatalogService - catalog = getCatalogService() - tableFilters = {} - if not featureDataSources: - selectedTables = catalog.getDataObjects(featureCode) - else: - allObjs = {o["meta"]["table"]: o for o in catalog.getDataObjects(featureCode) if "meta" in o and "table" in o.get("meta", {})} - selectedTables = [allObjs[ds["tableName"]] for ds in featureDataSources if ds.get("tableName") in allObjs] - for ds in featureDataSources: - rf = ds.get("recordFilter") - if rf and isinstance(rf, dict) and ds.get("tableName"): - tableFilters[ds["tableName"]] = rf - - if not selectedTables: - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error=f"No data tables available for feature '{featureCode}'", - ) - - from modules.connectors.connectorDbPostgre import DatabaseConnector - from modules.shared.configuration import APP_CONFIG - featureDbName = f"poweron_{featureCode.lower()}" - featureDbConn = DatabaseConnector( - dbHost=APP_CONFIG.get("DB_HOST", "localhost"), - dbDatabase=featureDbName, - dbUser=APP_CONFIG.get("DB_USER"), - dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"), - dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), - userId=userId or "agent", - ) - - aiService = services.ai if hasattr(services, "ai") else None - if aiService is None: - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error="AI service not available for sub-agent", - ) - - async def _subAgentAiCall(req): - if _anySourceNeutralize: - req.requireNeutralization = True - return await aiService.callAi(req) - - try: - answer = await runFeatureDataAgent( - question=question, - featureInstanceId=featureInstanceId, - featureCode=featureCode, - selectedTables=selectedTables, - mandateId=mandateId, - userId=userId, - aiCallFn=_subAgentAiCall, - dbConnector=featureDbConn, - instanceLabel=instanceLabel, - tableFilters=tableFilters, - ) - finally: - try: - featureDbConn.close() - except Exception: - pass - - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=True, data=answer, - ) - except Exception as e: - logger.error(f"queryFeatureInstance failed: {e}", exc_info=True) - return ToolResult( - toolCallId="", toolName="queryFeatureInstance", - success=False, error=str(e), - ) - - registry.register( - "queryFeatureInstance", _queryFeatureInstance, - description=( - "Query data from a feature instance (e.g. Trustee, CommCoach). " - "Delegates to a specialized sub-agent that knows the feature's data schema " - "and can browse/query its tables. Use this when the user has attached " - "feature data sources or asks about feature-specific data." - ), - parameters={ - "type": "object", - "properties": { - "featureInstanceId": {"type": "string", "description": "ID of the feature instance to query"}, - "question": {"type": "string", "description": "What data to find or analyze from this feature instance"}, - }, - "required": ["featureInstanceId", "question"] - }, - readOnly=True - ) - - # ---- Cross-workflow tools ---- - - async def _listWorkflowHistory(args: Dict[str, Any], context: Dict[str, Any]) -> ToolResult: - """List all chat workflows in this workspace with metadata.""" - import json as _json - try: - chatService = services.chat - chatInterface = chatService.interfaceDbChat - allWorkflows = chatInterface.getWorkflows() or [] - - allWorkflows.sort( - key=lambda w: w.get("sysCreatedAt") or w.get("startedAt") or 0, - reverse=True, - ) - allWorkflows = allWorkflows[:50] - - items = [] - for wf in allWorkflows: - wfId = wf.get("id", "") - name = wf.get("name") or "(unnamed)" - createdAt = wf.get("sysCreatedAt") or wf.get("startedAt") or 0 - lastActivity = wf.get("lastActivity") or createdAt - - msgs = chatInterface.getMessages(wfId) or [] - messageCount = len(msgs) - lastPreview = "" - if msgs: - lastMsg = msgs[-1] if isinstance(msgs[-1], dict) else ( - msgs[-1].model_dump() if hasattr(msgs[-1], "model_dump") else {} - ) - content = lastMsg.get("message") or lastMsg.get("content") or "" - lastPreview = content[:150] - - items.append({ - "id": wfId, - "name": name, - "createdAt": createdAt, - "lastActivity": lastActivity, - "messageCount": messageCount, - "lastMessagePreview": lastPreview, - }) - - return ToolResult( - toolCallId="", toolName="listWorkflowHistory", - success=True, data=_json.dumps(items, ensure_ascii=False), - ) - except Exception as e: - return ToolResult( - toolCallId="", toolName="listWorkflowHistory", - success=False, error=str(e), - ) - - registry.register( - "listWorkflowHistory", _listWorkflowHistory, - description=( - "List all chat conversations/workflows in this workspace. " - "Returns id, name, createdAt, lastActivity, messageCount, and a preview " - "of the last message for each workflow. Use this to discover previous " - "conversations when the user asks about past chats or wants a summary " - "across conversations." - ), - parameters={ - "type": "object", - "properties": {}, - }, - readOnly=True, - ) - - async def _readWorkflowMessages(args: Dict[str, Any], context: Dict[str, Any]) -> ToolResult: - """Read messages from a specific workflow.""" - import json as _json - targetWorkflowId = args.get("workflowId", "") - limit = int(args.get("limit", 20)) - offset = int(args.get("offset", 0)) - - if not targetWorkflowId: - return ToolResult( - toolCallId="", toolName="readWorkflowMessages", - success=False, error="workflowId is required", - ) - - try: - chatService = services.chat - chatInterface = chatService.interfaceDbChat - allMsgs = chatInterface.getMessages(targetWorkflowId) or [] - - sliced = allMsgs[offset:offset + limit] - items = [] - for msg in sliced: - raw = msg if isinstance(msg, dict) else ( - msg.model_dump() if hasattr(msg, "model_dump") else {} - ) - content = raw.get("message") or raw.get("content") or "" - if len(content) > 2000: - content = content[:2000] + "..." - items.append({ - "role": raw.get("role", ""), - "message": content, - "publishedAt": raw.get("publishedAt") or raw.get("sysCreatedAt") or 0, - }) - - header = f"Workflow {targetWorkflowId}: {len(allMsgs)} total messages" - if offset > 0 or len(allMsgs) > offset + limit: - header += f" (showing {offset + 1}-{offset + len(sliced)})" - - return ToolResult( - toolCallId="", toolName="readWorkflowMessages", - success=True, - data=header + "\n" + _json.dumps(items, ensure_ascii=False), - ) - except Exception as e: - return ToolResult( - toolCallId="", toolName="readWorkflowMessages", - success=False, error=str(e), - ) - - registry.register( - "readWorkflowMessages", _readWorkflowMessages, - description=( - "Read messages from a specific chat workflow/conversation. " - "Use this after listWorkflowHistory to read the content of a " - "specific past conversation. Supports pagination via offset/limit." - ), - parameters={ - "type": "object", - "properties": { - "workflowId": {"type": "string", "description": "ID of the workflow to read messages from"}, - "limit": {"type": "integer", "description": "Max messages to return (default 20)"}, - "offset": {"type": "integer", "description": "Skip first N messages (default 0)"}, - }, - "required": ["workflowId"], - }, - readOnly=True, - ) - - # Tag core-only tools so restricted toolSets (e.g. "commcoach") exclude them. - # Tools NOT in this set remain toolSet=None → available to ALL sets. - _CORE_ONLY_TOOLS = { - "listFiles", "listFolders", "tagFile", "moveFile", "createFolder", - "writeFile", "deleteFile", "renameFile", "translateText", - "deleteFolder", "renameFolder", "moveFolder", "copyFile", "replaceInFile", - "listConnections", "uploadToExternal", "sendMail", "downloadFromDataSource", - "browseContainer", "readContentObjects", "extractContainerItem", - "summarizeContent", "describeImage", "renderDocument", - "textToSpeech", "generateImage", "createChart", - "speechToText", "detectLanguage", "neutralizeData", "executeCode", - "listWorkflowHistory", "readWorkflowMessages", - } - for _toolName in _CORE_ONLY_TOOLS: - _td = registry.getTool(_toolName) - if _td: - _td.toolSet = "core" diff --git a/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py b/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py new file mode 100644 index 00000000..459c4cab --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py @@ -0,0 +1,183 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Toolbox Registry for the Agent service. +Manages thematic tool groupings (toolboxes) and the `requestToolbox` meta-tool. +""" + +import logging +from typing import Dict, List, Any, Optional, Set +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + + +class ToolboxDefinition(BaseModel): + """Definition of a thematic toolbox.""" + id: str = Field(description="Unique toolbox identifier (e.g. 'core', 'email', 'workflow')") + label: str = Field(description="Human-readable label") + description: str = Field(default="", description="What this toolbox provides") + featureCode: Optional[str] = Field(default=None, description="Feature code if toolbox is feature-specific") + tools: List[str] = Field(default_factory=list, description="Tool names belonging to this toolbox") + isDefault: bool = Field(default=False, description="If true, toolbox is active by default") + requiresConnection: Optional[str] = Field( + default=None, + description="Connection authority required (e.g. 'microsoft', 'google'). None = always available." + ) + + +class ToolboxRegistry: + """Registry for toolbox definitions. Manages activation and tool lookup.""" + + def __init__(self): + self._toolboxes: Dict[str, ToolboxDefinition] = {} + + def registerToolbox(self, toolbox: ToolboxDefinition) -> None: + """Register a toolbox definition.""" + if toolbox.id in self._toolboxes: + logger.debug("Toolbox '%s' already registered, updating", toolbox.id) + self._toolboxes[toolbox.id] = toolbox + logger.debug("Registered toolbox: %s (%d tools, default=%s)", toolbox.id, len(toolbox.tools), toolbox.isDefault) + + def getToolbox(self, toolboxId: str) -> Optional[ToolboxDefinition]: + """Get a toolbox by ID.""" + return self._toolboxes.get(toolboxId) + + def getAllToolboxes(self) -> List[ToolboxDefinition]: + """Get all registered toolboxes.""" + return list(self._toolboxes.values()) + + def getDefaultToolboxes(self) -> List[ToolboxDefinition]: + """Get all default toolboxes (active at agent start).""" + return [tb for tb in self._toolboxes.values() if tb.isDefault] + + def getActiveToolboxes(self, userConnections: List[str] = None) -> List[ToolboxDefinition]: + """ + Get toolboxes available to the user based on their connections. + Toolboxes without requiresConnection are always available. + Toolboxes with requiresConnection are available only if the user has that connection. + """ + available = [] + connectionAuthorities: Set[str] = set(userConnections or []) + for tb in self._toolboxes.values(): + if tb.requiresConnection is None: + available.append(tb) + elif tb.requiresConnection in connectionAuthorities: + available.append(tb) + return available + + def getToolsForToolboxes(self, toolboxIds: List[str]) -> List[str]: + """Get the union of all tool names for the given toolbox IDs.""" + tools: Set[str] = set() + for tbId in toolboxIds: + tb = self._toolboxes.get(tbId) + if tb: + tools.update(tb.tools) + return sorted(tools) + + def getToolboxForTool(self, toolName: str) -> Optional[str]: + """Find which toolbox a tool belongs to.""" + for tb in self._toolboxes.values(): + if toolName in tb.tools: + return tb.id + return None + + def toApiResponse(self, userConnections: List[str] = None) -> List[Dict[str, Any]]: + """Serialize available toolboxes for API response.""" + available = self.getActiveToolboxes(userConnections) + return [ + { + "id": tb.id, + "label": tb.label, + "description": tb.description, + "toolCount": len(tb.tools), + "isDefault": tb.isDefault, + "requiresConnection": tb.requiresConnection, + } + for tb in available + ] + + +# Module-level singleton +_toolboxRegistry = ToolboxRegistry() + + +def getToolboxRegistry() -> ToolboxRegistry: + """Get the global toolbox registry singleton.""" + return _toolboxRegistry + + +def _registerDefaultToolboxes() -> None: + """Register the default set of toolboxes.""" + defaults = [ + ToolboxDefinition( + id="core", + label="Core Tools", + description="Basic agent tools: search, read, write, web", + isDefault=True, + tools=[], + ), + ToolboxDefinition( + id="ai", + label="AI Tools", + description="AI-powered analysis and generation", + isDefault=True, + tools=[], + ), + ToolboxDefinition( + id="datasources", + label="Data Sources", + description="Access external data sources and databases", + isDefault=False, + tools=[], + ), + ToolboxDefinition( + id="email", + label="Email", + description="Read and send emails via Outlook/Gmail", + requiresConnection="microsoft", + isDefault=False, + tools=[], + ), + ToolboxDefinition( + id="sharepoint", + label="SharePoint", + description="Access SharePoint sites, lists, and files", + requiresConnection="microsoft", + isDefault=False, + tools=[], + ), + ToolboxDefinition( + id="clickup", + label="ClickUp", + description="Manage ClickUp tasks and projects", + requiresConnection="clickup", + isDefault=False, + tools=[], + ), + ToolboxDefinition( + id="jira", + label="Jira", + description="Manage Jira issues and projects", + requiresConnection="jira", + isDefault=False, + tools=[], + ), + ToolboxDefinition( + id="workflow", + label="Workflow", + description="Graph manipulation tools for the visual editor", + featureCode="graphicalEditor", + isDefault=False, + tools=[ + "readWorkflowGraph", "addNode", "removeNode", "connectNodes", + "setNodeParameter", "listAvailableNodeTypes", "validateGraph", + "listWorkflowHistory", "readWorkflowMessages", + ], + ), + ] + for tb in defaults: + _toolboxRegistry.registerToolbox(tb) + + +_registerDefaultToolboxes() diff --git a/modules/serviceCenter/services/serviceAgent/workflowTools.py b/modules/serviceCenter/services/serviceAgent/workflowTools.py new file mode 100644 index 00000000..a63abb65 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/workflowTools.py @@ -0,0 +1,479 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Workflow Toolbox - AI-assisted graph manipulation tools for the GraphicalEditor. +Tools: readWorkflowGraph, addNode, removeNode, connectNodes, setNodeParameter, + listAvailableNodeTypes, validateGraph, listWorkflowHistory, readWorkflowMessages. +""" + +import logging +import uuid +from typing import Dict, Any, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult + +logger = logging.getLogger(__name__) + +TOOLBOX_ID = "workflow" + + +async def _readWorkflowGraph(params: Dict[str, Any], context: Any) -> ToolResult: + """Read the current workflow graph (nodes and connections).""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + if not workflowId or not instanceId: + return ToolResult(success=False, error="workflowId and instanceId required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = wf.get("graph", {}) + nodes = graph.get("nodes", []) + connections = graph.get("connections", []) + return ToolResult( + success=True, + data={ + "workflowId": workflowId, + "label": wf.get("label", ""), + "nodeCount": len(nodes), + "connectionCount": len(connections), + "nodes": [{"id": n.get("id"), "type": n.get("type"), "title": n.get("title", "")} for n in nodes], + "connections": connections, + }, + ) + except Exception as e: + logger.exception("readWorkflowGraph failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _addNode(params: Dict[str, Any], context: Any) -> ToolResult: + """Add a node to the workflow graph.""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + nodeType = params.get("nodeType") + if not workflowId or not instanceId or not nodeType: + return ToolResult(success=False, error="workflowId, instanceId, and nodeType required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = dict(wf.get("graph", {})) + nodes = list(graph.get("nodes", [])) + + nodeId = params.get("nodeId") or str(uuid.uuid4())[:8] + title = params.get("title", "") + nodeParams = params.get("parameters", {}) + position = params.get("position", {"x": len(nodes) * 200, "y": 100}) + + newNode = { + "id": nodeId, + "type": nodeType, + "title": title, + "parameters": nodeParams, + "position": position, + } + nodes.append(newNode) + graph["nodes"] = nodes + + iface.updateWorkflow(workflowId, {"graph": graph}) + return ToolResult( + success=True, + data={"nodeId": nodeId, "nodeType": nodeType, "message": f"Node '{title or nodeType}' added"}, + ) + except Exception as e: + logger.exception("addNode failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _removeNode(params: Dict[str, Any], context: Any) -> ToolResult: + """Remove a node and its connections from the workflow graph.""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + nodeId = params.get("nodeId") + if not workflowId or not instanceId or not nodeId: + return ToolResult(success=False, error="workflowId, instanceId, and nodeId required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = dict(wf.get("graph", {})) + nodes = [n for n in graph.get("nodes", []) if n.get("id") != nodeId] + connections = [ + c for c in graph.get("connections", []) + if c.get("source") != nodeId and c.get("target") != nodeId + ] + graph["nodes"] = nodes + graph["connections"] = connections + + iface.updateWorkflow(workflowId, {"graph": graph}) + return ToolResult(success=True, data={"nodeId": nodeId, "message": f"Node {nodeId} removed"}) + except Exception as e: + logger.exception("removeNode failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _connectNodes(params: Dict[str, Any], context: Any) -> ToolResult: + """Connect two nodes in the workflow graph.""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + sourceId = params.get("sourceId") + targetId = params.get("targetId") + if not workflowId or not instanceId or not sourceId or not targetId: + return ToolResult(success=False, error="workflowId, instanceId, sourceId, and targetId required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = dict(wf.get("graph", {})) + connections = list(graph.get("connections", [])) + newConn = { + "source": sourceId, + "target": targetId, + "sourceOutput": params.get("sourceOutput", 0), + "targetInput": params.get("targetInput", 0), + } + connections.append(newConn) + graph["connections"] = connections + + iface.updateWorkflow(workflowId, {"graph": graph}) + return ToolResult(success=True, data={"connection": newConn, "message": f"Connected {sourceId} -> {targetId}"}) + except Exception as e: + logger.exception("connectNodes failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _setNodeParameter(params: Dict[str, Any], context: Any) -> ToolResult: + """Set a parameter on a node.""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + nodeId = params.get("nodeId") + paramName = params.get("parameterName") + paramValue = params.get("parameterValue") + if not workflowId or not instanceId or not nodeId or not paramName: + return ToolResult(success=False, error="workflowId, instanceId, nodeId, and parameterName required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = dict(wf.get("graph", {})) + nodes = list(graph.get("nodes", [])) + found = False + for n in nodes: + if n.get("id") == nodeId: + nodeParams = dict(n.get("parameters", {})) + nodeParams[paramName] = paramValue + n["parameters"] = nodeParams + found = True + break + + if not found: + return ToolResult(success=False, error=f"Node {nodeId} not found in graph") + + graph["nodes"] = nodes + iface.updateWorkflow(workflowId, {"graph": graph}) + return ToolResult(success=True, data={"nodeId": nodeId, "parameter": paramName, "message": f"Parameter '{paramName}' set"}) + except Exception as e: + logger.exception("setNodeParameter failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _listAvailableNodeTypes(params: Dict[str, Any], context: Any) -> ToolResult: + """List all available node types for the flow builder.""" + try: + from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES + nodeTypes = [ + {"id": n.get("id"), "category": n.get("category"), "label": n.get("label", {}).get("en", n.get("id"))} + for n in STATIC_NODE_TYPES + ] + return ToolResult(success=True, data={"nodeTypes": nodeTypes, "count": len(nodeTypes)}) + except Exception as e: + logger.exception("listAvailableNodeTypes failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _validateGraph(params: Dict[str, Any], context: Any) -> ToolResult: + """Validate a workflow graph for common issues.""" + try: + workflowId = params.get("workflowId") + instanceId = params.get("instanceId") + if not workflowId or not instanceId: + return ToolResult(success=False, error="workflowId and instanceId required") + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf: + return ToolResult(success=False, error=f"Workflow {workflowId} not found") + + graph = wf.get("graph", {}) + nodes = graph.get("nodes", []) + connections = graph.get("connections", []) + issues: List[str] = [] + + nodeIds = {n.get("id") for n in nodes} + if not nodes: + issues.append("Graph has no nodes") + + hasTrigger = any(n.get("type", "").startswith("trigger.") for n in nodes) + if not hasTrigger: + issues.append("No trigger node found") + + for c in connections: + if c.get("source") not in nodeIds: + issues.append(f"Connection source '{c.get('source')}' not found") + if c.get("target") not in nodeIds: + issues.append(f"Connection target '{c.get('target')}' not found") + + connectedNodes = set() + for c in connections: + connectedNodes.add(c.get("source")) + connectedNodes.add(c.get("target")) + orphans = [n.get("id") for n in nodes if n.get("id") not in connectedNodes and not n.get("type", "").startswith("trigger.")] + if orphans: + issues.append(f"Orphan nodes (not connected): {', '.join(orphans)}") + + return ToolResult( + success=True, + data={ + "valid": len(issues) == 0, + "issues": issues, + "nodeCount": len(nodes), + "connectionCount": len(connections), + }, + ) + except Exception as e: + logger.exception("validateGraph failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _listWorkflowHistory(params: Dict[str, Any], context: Any) -> ToolResult: + """List versions (history) for a workflow.""" + try: + workflowId = params.get("workflowId", "") + instanceId = params.get("instanceId", "") + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + versions = iface.getVersions(workflowId) + return ToolResult( + success=True, + data={ + "workflowId": workflowId, + "versions": [ + { + "id": v.get("id"), + "versionNumber": v.get("versionNumber"), + "status": v.get("status"), + "publishedAt": v.get("publishedAt"), + "publishedBy": v.get("publishedBy"), + } + for v in versions + ], + }, + ) + except Exception as e: + logger.exception("listWorkflowHistory failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +async def _readWorkflowMessages(params: Dict[str, Any], context: Any) -> ToolResult: + """Read recent run logs/messages for a workflow.""" + try: + workflowId = params.get("workflowId", "") + instanceId = params.get("instanceId", "") + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + user = getattr(context, "user", None) + mandateId = getattr(context, "mandateId", "") or "" + iface = getGraphicalEditorInterface(user, mandateId, instanceId) + from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoRun + runs = iface.db.getRecordset(AutoRun, recordFilter={"workflowId": workflowId}) or [] + runSummaries = [] + for r in sorted(runs, key=lambda x: x.get("startedAt") or 0, reverse=True)[:10]: + runSummaries.append({ + "runId": r.get("id"), + "status": r.get("status"), + "startedAt": r.get("startedAt"), + "completedAt": r.get("completedAt"), + "error": r.get("error"), + }) + return ToolResult( + success=True, + data={"workflowId": workflowId, "recentRuns": runSummaries}, + ) + except Exception as e: + logger.exception("readWorkflowMessages failed: %s", e) + return ToolResult(success=False, error=str(e)) + + +def getWorkflowToolDefinitions() -> List[Dict[str, Any]]: + """Return tool definitions for registration in the ToolRegistry.""" + return [ + { + "name": "readWorkflowGraph", + "handler": _readWorkflowGraph, + "description": "Read the current workflow graph (nodes and connections)", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string", "description": "Workflow ID"}, + "instanceId": {"type": "string", "description": "Feature instance ID"}, + }, + "required": ["workflowId", "instanceId"], + }, + "toolSet": TOOLBOX_ID, + }, + { + "name": "addNode", + "handler": _addNode, + "description": "Add a node to the workflow graph", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + "nodeType": {"type": "string", "description": "Node type (e.g. ai.chat, email.send)"}, + "title": {"type": "string", "description": "Human-readable title"}, + "parameters": {"type": "object", "description": "Node parameters"}, + "position": {"type": "object", "description": "Canvas position {x, y}"}, + }, + "required": ["workflowId", "instanceId", "nodeType"], + }, + "toolSet": TOOLBOX_ID, + }, + { + "name": "removeNode", + "handler": _removeNode, + "description": "Remove a node and its connections from the graph", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + "nodeId": {"type": "string", "description": "ID of the node to remove"}, + }, + "required": ["workflowId", "instanceId", "nodeId"], + }, + "toolSet": TOOLBOX_ID, + }, + { + "name": "connectNodes", + "handler": _connectNodes, + "description": "Connect two nodes in the graph", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + "sourceId": {"type": "string"}, + "targetId": {"type": "string"}, + "sourceOutput": {"type": "integer", "default": 0}, + "targetInput": {"type": "integer", "default": 0}, + }, + "required": ["workflowId", "instanceId", "sourceId", "targetId"], + }, + "toolSet": TOOLBOX_ID, + }, + { + "name": "setNodeParameter", + "handler": _setNodeParameter, + "description": "Set a parameter on a node", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + "nodeId": {"type": "string"}, + "parameterName": {"type": "string"}, + "parameterValue": {"description": "Value to set (any type)"}, + }, + "required": ["workflowId", "instanceId", "nodeId", "parameterName", "parameterValue"], + }, + "toolSet": TOOLBOX_ID, + }, + { + "name": "listAvailableNodeTypes", + "handler": _listAvailableNodeTypes, + "description": "List all available node types for the flow builder", + "parameters": {"type": "object", "properties": {}}, + "readOnly": True, + "toolSet": TOOLBOX_ID, + }, + { + "name": "validateGraph", + "handler": _validateGraph, + "description": "Validate a workflow graph for common issues", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + }, + "required": ["workflowId", "instanceId"], + }, + "readOnly": True, + "toolSet": TOOLBOX_ID, + }, + { + "name": "listWorkflowHistory", + "handler": _listWorkflowHistory, + "description": "List version history for a workflow (AutoVersion entries)", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + }, + "required": ["workflowId", "instanceId"], + }, + "readOnly": True, + "toolSet": TOOLBOX_ID, + }, + { + "name": "readWorkflowMessages", + "handler": _readWorkflowMessages, + "description": "Read recent run logs and status for a workflow", + "parameters": { + "type": "object", + "properties": { + "workflowId": {"type": "string"}, + "instanceId": {"type": "string"}, + }, + "required": ["workflowId", "instanceId"], + }, + "readOnly": True, + "toolSet": TOOLBOX_ID, + }, + ] diff --git a/modules/serviceCenter/services/serviceMessaging/subscriptions/subSubscriptionGraphicalEditorRunFailed.py b/modules/serviceCenter/services/serviceMessaging/subscriptions/subSubscriptionGraphicalEditorRunFailed.py new file mode 100644 index 00000000..2d77fd5b --- /dev/null +++ b/modules/serviceCenter/services/serviceMessaging/subscriptions/subSubscriptionGraphicalEditorRunFailed.py @@ -0,0 +1,71 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Subscription handler for GraphicalEditor workflow run failures. +Sends email notifications to subscribed users when a workflow run fails. +""" + +from typing import List +from modules.datamodels.datamodelMessaging import ( + MessagingEventParameters, + MessagingSubscriptionExecutionResult, + MessagingSubscriptionRegistration, + MessagingChannel, +) + + +def execute( + eventParameters: MessagingEventParameters, + registrations: List[MessagingSubscriptionRegistration], + messagingService, +) -> MessagingSubscriptionExecutionResult: + """ + Subscription function for GraphicalEditor run failures. + Sends email/SMS to registered users when a workflow run fails. + """ + triggerData = eventParameters.triggerData or {} + workflowId = triggerData.get("workflowId", "Unknown") + workflowLabel = triggerData.get("workflowLabel", workflowId) + runId = triggerData.get("runId", "Unknown") + error = triggerData.get("error", "Unknown error") + mandateId = triggerData.get("mandateId", "") + + emailRegistrations = [r for r in registrations if r.channel == MessagingChannel.EMAIL] + smsRegistrations = [r for r in registrations if r.channel == MessagingChannel.SMS] + + emailSubject = f"Workflow fehlgeschlagen: {workflowLabel}" + emailMessage = ( + f"Ein Workflow-Run ist fehlgeschlagen.\n\n" + f"Workflow: {workflowLabel}\n" + f"Workflow-ID: {workflowId}\n" + f"Run-ID: {runId}\n" + f"Fehler: {error}\n\n" + f"Bitte prüfen Sie den Workflow im Grafischen Editor." + ) + + smsMessage = f"Workflow '{workflowLabel}' fehlgeschlagen: {error[:100]}" + + messagesSent = 0 + + for reg in emailRegistrations: + sendResult = messagingService.sendMessage( + subject=emailSubject, + message=emailMessage, + registration=reg, + ) + if sendResult.success: + messagesSent += 1 + + for reg in smsRegistrations: + sendResult = messagingService.sendMessage( + subject="", + message=smsMessage, + registration=reg, + ) + if sendResult.success: + messagesSent += 1 + + return MessagingSubscriptionExecutionResult( + success=True, + messagesSent=messagesSent, + ) diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index a424c973..44467114 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -272,26 +272,6 @@ NAVIGATION_SECTIONS = [ "adminOnly": True, "sysAdminOnly": True, }, - { - "id": "admin-automation-events", - "objectKey": "ui.admin.automationEvents", - "label": {"en": "Automation Events", "de": "Automation Events", "fr": "Événements d'automatisation"}, - "icon": "FaClock", - "path": "/admin/automation-events", - "order": 80, - "adminOnly": True, - "sysAdminOnly": True, - }, - { - "id": "admin-automation-logs", - "objectKey": "ui.admin.automationLogs", - "label": {"en": "Execution Logs", "de": "Ausführungsprotokolle", "fr": "Journaux d'exécution"}, - "icon": "FaClipboardList", - "path": "/admin/automation-logs", - "order": 85, - "adminOnly": True, - "sysAdminOnly": True, - }, { "id": "admin-logs", "objectKey": "ui.admin.logs", @@ -369,7 +349,6 @@ UI_OBJECTS = _buildUiObjectsFromNavigation() # - data.uam.* → User Access Management (mandantenübergreifend) # - data.chat.* → Chat/AI-Daten (benutzer-eigen, kein Mandantenkontext) # - data.files.* → Dateien (benutzer-eigen) -# - data.automation.* → Automation (benutzer-eigen) # - data.feature.* → Mandanten-/Feature-spezifische Daten (dynamisch) # ============================================================================= @@ -437,12 +416,6 @@ DATA_OBJECTS = [ "label": {"en": "File", "de": "Datei", "fr": "Fichier"}, "meta": {"table": "FileItem", "namespace": "files", "groupDisabled": True} }, - # Automation - benutzer-eigen - { - "objectKey": "data.automation.AutomationDefinition", - "label": {"en": "Automation", "de": "Automatisierung", "fr": "Automatisation"}, - "meta": {"table": "AutomationDefinition", "namespace": "automation", "groupDisabled": True} - }, ] # ============================================================================= @@ -450,16 +423,6 @@ DATA_OBJECTS = [ # ============================================================================= RESOURCE_OBJECTS = [ - { - "objectKey": "resource.store.automation", - "label": {"en": "Store: Automation", "de": "Store: Automation", "fr": "Store: Automatisation"}, - "meta": {"category": "store", "featureCode": "automation"} - }, - { - "objectKey": "resource.store.automation2", - "label": {"en": "Store: Automation 2", "de": "Store: Automation 2", "fr": "Store: Automatisation 2"}, - "meta": {"category": "store", "featureCode": "automation2"} - }, { "objectKey": "resource.store.teamsbot", "label": {"en": "Store: Teams Bot", "de": "Store: Teams Bot", "fr": "Store: Teams Bot"}, diff --git a/modules/workflows/automation/__init__.py b/modules/workflows/automation/__init__.py deleted file mode 100644 index 89022240..00000000 --- a/modules/workflows/automation/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Workflow feature - handles workflow execution, scheduling, and start/stop operations. -""" - -from .mainWorkflow import chatStart, chatStop, executeAutomation, syncAutomationEvents, createAutomationEventHandler - -__all__ = ['chatStart', 'chatStop', 'executeAutomation', 'syncAutomationEvents', 'createAutomationEventHandler'] - diff --git a/modules/workflows/automation/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py deleted file mode 100644 index dc387926..00000000 --- a/modules/workflows/automation/mainWorkflow.py +++ /dev/null @@ -1,325 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Main workflow service - handles workflow execution, scheduling, and start/stop operations. -""" - -import logging -import json -from typing import Dict, Any, Optional - -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum -from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition -from modules.datamodels.datamodelUam import User -from modules.shared.timeUtils import getUtcTimestamp -from modules.shared.eventManagement import eventManager -from modules.features.automation.mainAutomation import getAutomationServices -from modules.workflows.workflowManager import WorkflowManager -from .subAutomationUtils import parseScheduleToCron, planToPrompt, replacePlaceholders - -logger = logging.getLogger(__name__) - - -async def chatStart(currentUser: User, userInput: UserInputRequest, workflowMode: WorkflowModeEnum, workflowId: Optional[str] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None, featureCode: Optional[str] = None, services=None) -> ChatWorkflow: - """ - Starts a new chat or continues an existing one, then launches processing asynchronously. - - Args: - currentUser: Current user - userInput: User input request - workflowId: Optional workflow ID to continue existing workflow - workflowMode: Workflow mode (Dynamic, Automation, etc.) - mandateId: Mandate ID (required for billing) - featureInstanceId: Feature instance ID (required for billing) - featureCode: Feature code (e.g., 'automation') - services: Pre-built service hub from the calling feature (required). Each feature must pass its own services. - """ - if services is None: - raise ValueError("services is required: each feature must pass its own service hub (e.g. getAutomationServices)") - try: - - # Store allowedProviders in services context for model selection - if hasattr(userInput, 'allowedProviders') and userInput.allowedProviders: - services.allowedProviders = userInput.allowedProviders - logger.info(f"AI provider filter active: {userInput.allowedProviders}") - - # Store feature code in services (for billing) - if featureCode: - services.featureCode = featureCode - - workflowManager = WorkflowManager(services) - workflow = await workflowManager.workflowStart(userInput, workflowMode, workflowId) - return workflow - except Exception as e: - logger.error(f"Error starting chat: {str(e)}") - raise - -async def chatStop(currentUser: User, workflowId: str, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None, featureCode: Optional[str] = None, services=None) -> ChatWorkflow: - """Stops a running chat. Caller must pass services from the owning feature.""" - if services is None: - raise ValueError("services is required: each feature must pass its own service hub (e.g. getAutomationServices)") - try: - if featureCode: - services.featureCode = featureCode - workflowManager = WorkflowManager(services) - return await workflowManager.workflowStop(workflowId) - except Exception as e: - logger.error(f"Error stopping chat: {str(e)}") - raise - - -async def executeAutomation(automationId: str, automation, creatorUser: User, services) -> ChatWorkflow: - """Execute automation workflow with the creator user's context. - - The automation object and creatorUser are resolved by the caller (handler) - using the SysAdmin eventUser. This function does NOT re-load them. - - Args: - automationId: ID of automation to execute - automation: Pre-loaded automation object (with system fields like sysCreatedBy) - creatorUser: The user who created the automation (workflow runs in this context) - services: Services instance (used for interfaceDbApp etc.) - - Returns: - ChatWorkflow instance created by automation execution - """ - executionStartTime = getUtcTimestamp() - executionLog = { - "timestamp": executionStartTime, - "workflowId": None, - "status": "running", - "messages": [] - } - - try: - executionLog["messages"].append(f"Started execution at {executionStartTime}") - - # Store allowed providers from automation in services context - if hasattr(automation, 'allowedProviders') and automation.allowedProviders: - services.allowedProviders = automation.allowedProviders - logger.debug(f"Automation {automationId} restricted to providers: {automation.allowedProviders}") - - # Context comes EXCLUSIVELY from the automation definition - automationMandateId = str(automation.mandateId) if automation.mandateId is not None else None - automationFeatureInstanceId = str(automation.featureInstanceId) if automation.featureInstanceId is not None else None - - if not automationMandateId or not automationFeatureInstanceId: - raise ValueError(f"Automation {automationId} missing mandateId or featureInstanceId") - - logger.info(f"Executing automation {automationId} as user {creatorUser.id} with mandateId={automationMandateId}, featureInstanceId={automationFeatureInstanceId}") - - # 1. Replace placeholders in template to generate plan - template = automation.template or "" - placeholders = automation.placeholders or {} - planJson = replacePlaceholders(template, placeholders) - try: - plan = json.loads(planJson) - except json.JSONDecodeError as e: - logger.error(f"Failed to parse plan JSON after placeholder replacement: {str(e)}") - logger.error(f"Template: {template[:500]}...") - logger.error(f"Placeholders: {placeholders}") - logger.error(f"Generated planJson (first 1000 chars): {planJson[:1000]}") - logger.error(f"Error position: line {e.lineno}, column {e.colno}, char {e.pos}") - if e.pos is not None: - start = max(0, e.pos - 100) - end = min(len(planJson), e.pos + 100) - logger.error(f"Context around error: ...{planJson[start:end]}...") - raise ValueError(f"Invalid JSON after placeholder replacement: {str(e)}") - executionLog["messages"].append("Template placeholders replaced successfully") - executionLog["messages"].append(f"Using creator user: {creatorUser.id}") - - # 2. Create UserInputRequest from plan - # Embed plan JSON in prompt for TemplateMode to extract - promptText = planToPrompt(plan) - planJsonStr = json.dumps(plan) - # Embed plan as JSON comment so TemplateMode can extract it - promptWithPlan = f"{promptText}\n\n\n{planJsonStr}\n" - - userInput = UserInputRequest( - prompt=promptWithPlan, - listFileId=[], - userLanguage=creatorUser.language or "en" - ) - - executionLog["messages"].append("Starting workflow execution") - - # 3. Start workflow using chatStart with creator's context - # mandateId and featureInstanceId come from the automation definition - # Each feature must pass its own services - no fallback - creatorServices = getAutomationServices( - creatorUser, - mandateId=automationMandateId, - featureInstanceId=automationFeatureInstanceId, - ) - workflow = await chatStart( - currentUser=creatorUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_AUTOMATION, - workflowId=None, - mandateId=automationMandateId, - featureInstanceId=automationFeatureInstanceId, - featureCode='automation', - services=creatorServices, - ) - - executionLog["workflowId"] = workflow.id - executionLog["status"] = "completed" - executionLog["messages"].append(f"Workflow {workflow.id} started successfully") - logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)") - - # Set workflow name with "automated" prefix — use creatorServices from chatStart - automationLabel = automation.label or "Unknown Automation" - workflowName = f"automated: {automationLabel}" - creatorServices.interfaceDbChat.updateWorkflow(workflow.id, {"name": workflowName}) - logger.info(f"Set workflow {workflow.id} name to: {workflowName}") - - # Save execution log (bypasses RBAC — system operation, not a user edit) - executionLogs = list(automation.executionLogs or []) - executionLogs.append(executionLog) - # Keep only last 50 executions - if len(executionLogs) > 50: - executionLogs = executionLogs[-50:] - - services.interfaceDbAutomation._saveExecutionLog(automationId, executionLogs) - - return workflow - except Exception as e: - # Log error to execution log - executionLog["status"] = "error" - executionLog["messages"].append(f"Error: {str(e)}") - - # Save execution log even on error (bypasses RBAC — system operation) - # Use the automation object already passed in (no re-load needed) - try: - executionLogs = list(getattr(automation, 'executionLogs', None) or []) - executionLogs.append(executionLog) - if len(executionLogs) > 50: - executionLogs = executionLogs[-50:] - services.interfaceDbAutomation._saveExecutionLog(automationId, executionLogs) - except Exception as logError: - logger.error(f"Error saving execution log: {str(logError)}") - - raise - - -def syncAutomationEvents(services, eventUser) -> Dict[str, Any]: - """Sync scheduler with all active automations. - All operations (DB reads, scheduler registration) are synchronous. - - Args: - services: Services instance for data access - eventUser: System-level event user for accessing automations - - Returns: - Dictionary with sync results (synced count and event IDs) - """ - # Get all automation definitions filtered by RBAC (for current mandate) - filtered = services.interfaceDbAutomation.getAllAutomationDefinitionsWithRBAC(eventUser) - - registeredEvents = {} - - for automation in filtered: - # Handle both dict and object access patterns - if isinstance(automation, dict): - automationId = automation.get('id') - isActive = automation.get('active', False) - currentEventId = automation.get('eventId') - schedule = automation.get('schedule') - else: - automationId = automation.id - isActive = automation.active if hasattr(automation, 'active') else False - currentEventId = automation.eventId if hasattr(automation, 'eventId') else None - schedule = automation.schedule if hasattr(automation, 'schedule') else None - - if not schedule: - logger.warning(f"Automation {automationId} has no schedule, skipping") - continue - - try: - # Parse schedule to cron kwargs - cronKwargs = parseScheduleToCron(schedule) - - if isActive: - newEventId = f"automation.{automationId}" - handler = createAutomationEventHandler(automationId, eventUser) - - # Register with replaceExisting=True (atomically replaces old event) - eventManager.registerCron( - jobId=newEventId, - func=handler, - cronKwargs=cronKwargs, - replaceExisting=True - ) - - # Update automation with new eventId - if currentEventId != newEventId: - services.interfaceDbAutomation.updateAutomationDefinition( - automationId, - {"eventId": newEventId} - ) - - registeredEvents[automationId] = newEventId - else: - # Remove event if exists - if currentEventId: - try: - eventManager.remove(currentEventId) - services.interfaceDbAutomation.updateAutomationDefinition( - automationId, - {"eventId": None} - ) - except Exception as e: - logger.warning(f"Error removing event {currentEventId}: {str(e)}") - except Exception as e: - logger.error(f"Error syncing automation {automationId}: {str(e)}") - - return { - "synced": len(registeredEvents), - "events": registeredEvents - } - - -def createAutomationEventHandler(automationId: str, eventUser): - """Create event handler function for a specific automation. - - Args: - automationId: ID of automation to create handler for - eventUser: System-level event user for accessing automations (captured in closure) - - Returns: - Async handler function for scheduled automation execution - """ - async def handler(): - try: - if not eventUser: - logger.error("Event user not available for automation execution") - return - - # Load automation using SysAdmin eventUser (has unrestricted access) - eventServices = getAutomationServices(eventUser, mandateId=None, featureInstanceId=None) - automation = eventServices.interfaceDbAutomation.getAutomationDefinition(automationId, includeSystemFields=True) - if not automation or not getattr(automation, "active", False): - logger.warning(f"Automation {automationId} not found or not active, skipping execution") - return - - # Get creator user ID from automation's sysCreatedBy system field - creatorUserId = getattr(automation, "sysCreatedBy", None) - if not creatorUserId: - logger.error(f"Automation {automationId} has no creator user (sysCreatedBy missing)") - return - - # Get creator user from database (using SysAdmin access) - creatorUser = eventServices.interfaceDbApp.getUser(creatorUserId) - if not creatorUser: - logger.error(f"Creator user {creatorUserId} not found for automation {automationId}") - return - - # Execute automation — pass automation object and creatorUser directly - # No re-load needed in executeAutomation - await executeAutomation(automationId, automation, creatorUser, eventServices) - logger.info(f"Successfully executed automation {automationId} as user {creatorUserId}") - except Exception as e: - logger.error(f"Error executing automation {automationId}: {str(e)}") - - return handler - diff --git a/modules/workflows/automation/subAutomationSchedule.py b/modules/workflows/automation/subAutomationSchedule.py deleted file mode 100644 index 18cc4245..00000000 --- a/modules/workflows/automation/subAutomationSchedule.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Automation Lifecycle Manager. -Handles startup and shutdown of scheduled automations. - -Note: This module is NOT for feature container lifecycle - it only manages -the automation scheduler (loading/syncing scheduled automation events). -""" - -import logging -from modules.features.automation.mainAutomation import getAutomationServices - -logger = logging.getLogger(__name__) - - -def start(eventUser) -> bool: - """ - Start automation scheduler and sync scheduled events. - All operations are synchronous (DB access, scheduler registration). - - Args: - eventUser: System-level event user for background operations (provided by app.py) - """ - if not eventUser: - logger.warning("Automation: No event user provided, skipping automation sync") - return True - - try: - from modules.workflows.automation import syncAutomationEvents - from modules.shared.callbackRegistry import callbackRegistry - - # Get services for event user (provides access to interfaces) - services = getAutomationServices(eventUser, mandateId=None, featureInstanceId=None) - - # Register callback for automation changes - def onAutomationChanged(chatInterface): - """Callback triggered when automations are created/updated/deleted.""" - eventServices = getAutomationServices(eventUser, mandateId=None, featureInstanceId=None) - syncAutomationEvents(eventServices, eventUser) - - callbackRegistry.register('automation.changed', onAutomationChanged) - logger.info("Automation: Registered change callback") - - # Initial sync on startup - syncAutomationEvents(services, eventUser) - logger.info("Automation: Scheduled events synced on startup") - - except Exception as e: - logger.error(f"Automation: Error setting up events on startup: {str(e)}") - return False - - return True - - -def stop(eventUser) -> bool: - """ - Stop automation scheduler. - - Args: - eventUser: System-level event user (provided by app.py) - """ - # Callbacks will remain registered (acceptable for shutdown) - logger.info("Automation: Scheduler stopped (callbacks cleaned up on shutdown)") - return True diff --git a/modules/workflows/automation/subAutomationTemplates.py b/modules/workflows/automation/subAutomationTemplates.py deleted file mode 100644 index eb131f0a..00000000 --- a/modules/workflows/automation/subAutomationTemplates.py +++ /dev/null @@ -1,385 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Automation templates for workflow definitions. - -Contains predefined workflow templates that can be used to create automation definitions. -""" - -from typing import Dict, Any - -# Automation templates structure -AUTOMATION_TEMPLATES: Dict[str, Any] = { - "sets": [ - { - "template": { - "overview": "SharePoint Themen Zusammenfassung", - "tasks": [ - { - "id": "Task01", - "title": "SharePoint Themen Zusammenfassung", - "description": "Erstellt eine Zusammenfassung aller SharePoint Sites und deren Inhalte", - "objective": "Erstelle eine Zusammenfassung aller SharePoint Themen (Sites) und deren Inhalte als Word-Dokument", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findDocumentPath", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "searchQuery": "*", - "maxResults": 100 - }, - "execResultLabel": "sharepoint_sites_found" - }, - { - "execMethod": "sharepoint", - "execAction": "listDocuments", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "pathQuery": "{{KEY:sharepointBasePath}}", - "includeSubfolders": True - }, - "execResultLabel": "sharepoint_structure" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:summaryPrompt}}", - "documentList": ["sharepoint_sites_found", "sharepoint_structure"], - "resultType": "docx" - }, - "execResultLabel": "sharepoint_summary" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["sharepoint_summary"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "sharepoint_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointBasePath": "/sites/company-share", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "summaryPrompt": "Erstelle eine umfassende Zusammenfassung aller SharePoint Sites und deren Inhalte. Strukturiere das Dokument nach Sites und fasse für jede Site die wichtigsten Themen, Ordnerstrukturen und Dokumente zusammen. Erstelle ein professionelles Word-Dokument mit Überschriften, Abschnitten und einer klaren Gliederung. Berücksichtige alle gefundenen Sites, deren Ordnerstrukturen und dokumentiere die wichtigsten Inhalte pro Site." - } - }, - { - "template": { - "overview": "Immobilienrecherche Zürich", - "tasks": [ - { - "id": "Task02", - "title": "Immobilienrecherche Zürich", - "description": "Webrecherche nach Immobilien im Kanton Zürich und Speicherung in Excel", - "objective": "Immobilienrecherche im Kanton Zürich zum Verkauf (5-20 Mio. CHF) und speichere Ergebnisse in Excel-Liste auf SharePoint", - "actionList": [ - { - "execMethod": "ai", - "execAction": "webResearch", - "execParameters": { - "prompt": "{{KEY:immobilienResearchPrompt}}", - "urlList": ["{{KEY:immobilienResearchUrl}}"] - }, - "execResultLabel": "immobilien_research_results" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:excelFormatPrompt}}", - "documentList": ["immobilien_research_results"], - "resultType": "xlsx" - }, - "execResultLabel": "immobilien_excel_list" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["immobilien_excel_list"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "immobilien_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "immobilienResearchUrl": ["https://www.homegate.ch", "https://www.immoscout24.ch", "https://www.immowelt.ch"], - "immobilienResearchPrompt": "Suche nach Immobilien zum Verkauf im Kanton Zürich, Schweiz, im Preisbereich von 5-20 Millionen CHF. Sammle Informationen zu: Ort, Preis, Beschreibung, URL zu Bildern, Verkäufer/Kontaktinformationen.", - "excelFormatPrompt": "Erstelle eine Excel-Datei mit den recherchierten Immobilien. Jede Immobilie soll eine Zeile sein mit den folgenden Spalten: Ort, Preis (in CHF), Beschreibung, URL zu Bild, Verkäufer. Verwende die Daten aus der Webrecherche." - } - }, - { - "template": { - "overview": "Spesenbelege Zusammenfassung", - "tasks": [ - { - "id": "Task03", - "title": "Spesenbelege CSV Zusammenfassung", - "description": "Liest PDF-Spesenbelege aus SharePoint-Ordner und erstellt CSV-Zusammenfassung", - "objective": "Extrahiere alle PDF-Spesenbelege aus einem SharePoint-Ordner und erstelle eine CSV-Datei mit allen Spesendaten im selben Ordner", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findDocumentPath", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "searchQuery": "{{KEY:sharepointFolderNameSource}}:files:.pdf", - "maxResults": 100 - }, - "execResultLabel": "sharepoint_pdf_files" - }, - { - "execMethod": "sharepoint", - "execAction": "readDocuments", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "pathObject": "sharepoint_pdf_files" - }, - "execResultLabel": "spesenbelege_documents" - }, - { - "execMethod": "ai", - "execAction": "process", - "execParameters": { - "aiPrompt": "{{KEY:expenseExtractionPrompt}}", - "documentList": ["spesenbelege_documents"], - "resultType": "csv" - }, - "execResultLabel": "spesenbelege_csv" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadDocument", - "execParameters": { - "connectionReference": "{{KEY:connectionName}}", - "documentList": ["spesenbelege_csv"], - "pathQuery": "{{KEY:sharepointFolderNameDestination}}" - }, - "execResultLabel": "spesenbelege_upload_result" - } - ] - } - ] - }, - "parameters": { - "connectionName": "connection:msft:p.motsch@valueon.ch", - "sharepointFolderNameSource": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/expenses", - "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", - "expenseExtractionPrompt": "Verarbeite alle bereitgestellten Dokumente, aber extrahiere nur Daten aus PDF-Spesenbelegen (ignoriere andere Dateitypen). Für jeden gefundenen PDF-Spesenbeleg extrahiere als separaten Datensatz: Datum, Betrag, MWST %, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler/Verkäufer, Steuerbetrag. Erstelle eine CSV-Datei mit einer Zeile pro Spesenbeleg. Verwende die folgenden Spaltenüberschriften: Datum, Betrag, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler, Steuerbetrag. Stelle sicher, dass alle Beträge numerisch sind und Datumswerte im Format YYYY-MM-DD vorliegen. Wenn ein Dokument kein Spesenbeleg ist, ignoriere es." - } - }, - { - "template": { - "overview": "Preprocessing Server Data Update", - "tasks": [ - { - "id": "Task04", - "title": "Trigger Preprocessing Server", - "description": "Triggers the preprocessing server at customer tenant to update database with configuration", - "objective": "Call preprocessing server endpoint to update database with provided configuration JSON", - "actionList": [ - { - "execMethod": "context", - "execAction": "triggerPreprocessingServer", - "execParameters": { - "endpoint": "{{KEY:endpoint}}", - "configJson": "{{KEY:configJson}}", - "authSecretConfigKey": "{{KEY:authSecretConfigKey}}" - }, - "execResultLabel": "preprocessing_server_result" - } - ] - } - ] - }, - "parameters": { - "endpoint": "https://poweron-althaus-preprocess-prod-e3fegaatc7faency.switzerlandnorth-01.azurewebsites.net/api/v1/dataprocessor/update-db-with-config", - "authSecretConfigKey": "PREPROCESS_ALTHAUS_CHAT_SECRET", - "configJson": "{\"tables\":[{\"name\":\"Artikel\",\"powerbi_table_name\":\"Artikel\",\"steps\":[{\"keep\":{\"columns\":[\"I_ID\",\"Artikelbeschrieb\",\"Artikelbezeichnung\",\"Artikelgruppe\",\"Artikelkategorie\",\"Artikelkürzel\",\"Artikelnummer\",\"Einheit\",\"Gesperrt\",\"Keywords\",\"Lieferant\",\"Warengruppe\"]}},{\"fillna\":{\"column\":\"Lieferant\",\"value\":\"Unbekannt\"}}]},{\"name\":\"Einkaufspreis\",\"powerbi_table_name\":\"Einkaufspreis\",\"steps\":[{\"to_numeric\":{\"column\":\"EP_CHF\",\"errors\":\"coerce\"}},{\"dropna\":{\"subset\":[\"EP_CHF\"]}}]}]}" - } - }, - { - "template": { - "overview": "JIRA to SharePoint Ticket Synchronization", - "tasks": [ - { - "id": "Task01", - "title": "Sync JIRA Tickets to SharePoint", - "description": "Export JIRA tickets, merge with SharePoint file, upload back, and import changes to JIRA", - "objective": "Synchronize JIRA tickets with SharePoint file (bidirectional sync)", - "actionList": [ - { - "execMethod": "sharepoint", - "execAction": "findSiteByUrl", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "hostname": "{{KEY:sharepointHostname}}", - "sitePath": "{{KEY:sharepointSitePath}}" - }, - "execResultLabel": "sharepoint_site" - }, - { - "execMethod": "jira", - "execAction": "connectJira", - "execParameters": { - "apiUsername": "{{KEY:jiraUsername}}", - "apiTokenConfigKey": "{{KEY:jiraTokenConfigKey}}", - "apiUrl": "{{KEY:jiraUrl}}", - "projectCode": "{{KEY:jiraProjectCode}}", - "issueType": "{{KEY:jiraIssueType}}", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "jira_connection" - }, - { - "execMethod": "jira", - "execAction": "exportTicketsAsJson", - "execParameters": { - "connectionId": "jira_connection", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "jira_exported_tickets" - }, - { - "execMethod": "sharepoint", - "execAction": "downloadFileByPath", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" - }, - "execResultLabel": "existing_file_content" - }, - { - "execMethod": "jira", - "execAction": "parseExcelContent", - "execParameters": { - "excelContent": "existing_file_content", - "skipRows": 3, - "hasCustomHeaders": True - }, - "execResultLabel": "existing_parsed_data" - }, - { - "execMethod": "jira", - "execAction": "mergeTicketData", - "execParameters": { - "jiraData": "jira_exported_tickets", - "existingData": "existing_parsed_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}", - "idField": "ID" - }, - "execResultLabel": "merged_ticket_data" - }, - { - "execMethod": "sharepoint", - "execAction": "copyFile", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "sourceFolder": "{{KEY:sharepointMainFolder}}", - "sourceFile": "{{KEY:syncFileName}}", - "destFolder": "{{KEY:sharepointBackupFolder}}", - "destFile": "backup_{{TIMESTAMP}}_{{KEY:syncFileName}}" - }, - "execResultLabel": "file_backup" - }, - { - "execMethod": "jira", - "execAction": "createExcelContent", - "execParameters": { - "data": "merged_ticket_data", - "headers": "existing_parsed_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "new_file_content" - }, - { - "execMethod": "sharepoint", - "execAction": "uploadFile", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "folderPath": "{{KEY:sharepointMainFolder}}", - "fileName": "{{KEY:syncFileName}}", - "content": "new_file_content" - }, - "execResultLabel": "uploaded_file" - }, - { - "execMethod": "sharepoint", - "execAction": "downloadFileByPath", - "execParameters": { - "connectionReference": "{{KEY:sharepointConnection}}", - "siteId": "sharepoint_site", - "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" - }, - "execResultLabel": "uploaded_file_content" - }, - { - "execMethod": "jira", - "execAction": "parseExcelContent", - "execParameters": { - "excelContent": "uploaded_file_content", - "skipRows": 3, - "hasCustomHeaders": True - }, - "execResultLabel": "import_data" - }, - { - "execMethod": "jira", - "execAction": "importTicketsFromJson", - "execParameters": { - "connectionId": "jira_connection", - "ticketData": "import_data", - "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" - }, - "execResultLabel": "import_result" - } - ] - } - ] - }, - "parameters": { - "sharepointConnection": "connection:msft:patrick.motsch@delta.ch", - "sharepointHostname": "deltasecurityag.sharepoint.com", - "sharepointSitePath": "SteeringBPM", - "sharepointMainFolder": "/General/50 Docs hosted by SELISE", - "sharepointBackupFolder": "/General/50 Docs hosted by SELISE/SyncHistory", - "syncFileName": "DELTAgroup x SELISE Ticket Exchange List.xlsx", - "jiraUsername": "p.motsch@valueon.ch", - "jiraTokenConfigKey": "Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET", - "jiraUrl": "https://deltasecurity.atlassian.net", - "jiraProjectCode": "DCS", - "jiraIssueType": "Task", - "taskSyncDefinition": "{\"ID\":[\"get\",[\"key\"]],\"Module Category\":[\"get\",[\"fields\",\"customfield_10058\",\"value\"]],\"Summary\":[\"get\",[\"fields\",\"summary\"]],\"Description\":[\"get\",[\"fields\",\"description\"]],\"References\":[\"get\",[\"fields\",\"customfield_10066\"]],\"Priority\":[\"get\",[\"fields\",\"priority\",\"name\"]],\"Issue Status\":[\"get\",[\"fields\",\"status\",\"name\"]],\"Assignee\":[\"get\",[\"fields\",\"assignee\",\"displayName\"]],\"Issue Created\":[\"get\",[\"fields\",\"created\"]],\"Due Date\":[\"get\",[\"fields\",\"duedate\"]],\"DELTA Comments\":[\"get\",[\"fields\",\"customfield_10167\"]],\"SELISE Ticket References\":[\"put\",[\"fields\",\"customfield_10067\"]],\"SELISE Status Values\":[\"put\",[\"fields\",\"customfield_10065\"]],\"SELISE Comments\":[\"put\",[\"fields\",\"customfield_10168\"]]}" - } - } - ] -} - - -def getAutomationTemplates() -> Dict[str, Any]: - """ - Get automation templates. - - Returns: - Dict containing the automation templates structure with 'sets' key. - """ - return AUTOMATION_TEMPLATES - diff --git a/modules/workflows/automation/subAutomationUtils.py b/modules/workflows/automation/subAutomationUtils.py deleted file mode 100644 index bdac6efb..00000000 --- a/modules/workflows/automation/subAutomationUtils.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Utility functions for automation feature. - -Moved from interfaces/interfaceDbChat.py. -""" - -import json -from typing import Dict, Any -from datetime import datetime, UTC - - -def parseScheduleToCron(schedule: str) -> Dict[str, Any]: - """Parse schedule string to cron kwargs for APScheduler""" - parts = schedule.split() - if len(parts) != 5: - raise ValueError(f"Invalid schedule format: {schedule}") - - return { - "minute": parts[0], - "hour": parts[1], - "day": parts[2], - "month": parts[3], - "day_of_week": parts[4] - } - - -def planToPrompt(plan: Dict) -> str: - """Convert plan structure to prompt string for workflow execution""" - return plan.get("userMessage", plan.get("overview", "Execute automation workflow")) - - -def replacePlaceholders(template: str, placeholders: Dict[str, str]) -> str: - """Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}} or {{TIMESTAMP}}""" - result = template - - # Replace TIMESTAMP placeholder first (calculated placeholder, not from parameters) - timestampPattern = "{{TIMESTAMP}}" - if timestampPattern in result: - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") - result = result.replace(timestampPattern, timestamp) - - for placeholderName, value in placeholders.items(): - pattern = f"{{{{KEY:{placeholderName}}}}}" - - # Check if placeholder is in an array context like ["{{KEY:...}}"] - # If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array - arrayPattern = f'["{pattern}"]' - if arrayPattern in result: - # Check if value is a JSON array/dict - isArrayValue = False - arrayValue = None - - if isinstance(value, (list, dict)): - isArrayValue = True - arrayValue = json.dumps(value) - elif isinstance(value, str): - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - isArrayValue = True - arrayValue = value # Already valid JSON string - except (json.JSONDecodeError, ValueError): - pass - - if isArrayValue: - # Replace ["{{KEY:...}}"] with the array value - result = result.replace(arrayPattern, arrayValue) - continue # Skip the regular replacement below - - # Replace occurrences one-by-one to handle mixed contexts - while pattern in result: - patternStart = result.find(pattern) - isQuoted = False - if patternStart > 0: - charBefore = result[patternStart - 1] - patternEnd = patternStart + len(pattern) - charAfter = result[patternEnd] if patternEnd < len(result) else None - if charBefore == '"' and charAfter == '"': - isQuoted = True - - if isinstance(value, (list, dict)): - replacement = json.dumps(value) - elif isinstance(value, str): - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - else: - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - except (json.JSONDecodeError, ValueError): - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - else: - replacement = str(value) - result = result[:patternStart] + replacement + result[patternStart + len(pattern):] - return result - diff --git a/modules/workflows/automation2/executionEngine.py b/modules/workflows/automation2/executionEngine.py index 3ab08992..67344206 100644 --- a/modules/workflows/automation2/executionEngine.py +++ b/modules/workflows/automation2/executionEngine.py @@ -1,7 +1,10 @@ # Copyright (c) 2025 Patrick Motsch # Main execution engine for automation2 graphs. +import asyncio import logging +import time +import uuid from datetime import datetime, timezone from typing import Dict, Any, List, Set, Optional @@ -22,7 +25,7 @@ from modules.workflows.automation2.executors import ( PauseForHumanTaskError, PauseForEmailWaitError, ) -from modules.features.automation2.nodeDefinitions import STATIC_NODE_TYPES +from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES from modules.workflows.automation2.runEnvelope import normalize_run_envelope logger = logging.getLogger(__name__) @@ -82,6 +85,82 @@ def _getExecutor( return None +def _createStepLog(iface, runId: str, nodeId: str, nodeType: str, status: str = "running", inputSnapshot: Dict = None) -> Optional[str]: + """Create an AutoStepLog entry. Returns the step log ID or None if interface unavailable.""" + if not iface or not runId: + return None + try: + from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoStepLog + stepId = str(uuid.uuid4()) + iface.db.recordCreate(AutoStepLog, { + "id": stepId, + "runId": runId, + "nodeId": nodeId, + "nodeType": nodeType, + "status": status, + "inputSnapshot": inputSnapshot or {}, + "startedAt": time.time(), + }) + return stepId + except Exception as e: + logger.debug("Could not create AutoStepLog: %s", e) + return None + + +def _updateStepLog(iface, stepId: str, status: str, output: Dict = None, error: str = None, + durationMs: int = None, tokensUsed: int = 0, retryCount: int = 0) -> None: + """Update an AutoStepLog entry with results.""" + if not iface or not stepId: + return + try: + from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoStepLog + updates: Dict[str, Any] = { + "status": status, + "completedAt": time.time(), + } + if output is not None: + updates["output"] = output + if error is not None: + updates["error"] = error + if durationMs is not None: + updates["durationMs"] = durationMs + if tokensUsed: + updates["tokensUsed"] = tokensUsed + if retryCount: + updates["retryCount"] = retryCount + iface.db.recordModify(AutoStepLog, stepId, updates) + except Exception as e: + logger.debug("Could not update AutoStepLog %s: %s", stepId, e) + + +async def _executeWithRetry(executor, node, context, maxRetries: int = 0, retryDelaySeconds: float = 1.0): + """Execute a node with optional retry policy from node parameters.""" + params = node.get("parameters") or {} + retries = params.get("retryMaxAttempts", maxRetries) + delay = params.get("retryDelaySeconds", retryDelaySeconds) + attempt = 0 + lastError = None + while attempt <= retries: + try: + result = await executor.execute(node, context) + return result, attempt + except (PauseForHumanTaskError, PauseForEmailWaitError): + raise + except Exception as e: + lastError = e + attempt += 1 + if attempt <= retries: + logger.warning( + "Node %s failed (attempt %d/%d), retrying in %.1fs: %s", + node.get("id"), attempt, retries + 1, delay, e, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 60) + else: + raise lastError + raise lastError + + async def executeGraph( graph: Dict[str, Any], services: Any, @@ -202,17 +281,31 @@ async def executeGraph( if not executor: nodeOutputs[bnid] = None continue + _rStepStart = time.time() + _rStepId = _createStepLog(automation2_interface, runId, bnid, body_node.get("type", ""), "running") try: - result = await executor.execute(body_node, context) + result, _rRetry = await _executeWithRetry(executor, body_node, context) nodeOutputs[bnid] = result - logger.info("executeGraph loop resume body node %s done (iter %d)", bnid, next_index) + _rDur = int((time.time() - _rStepStart) * 1000) + _updateStepLog(automation2_interface, _rStepId, "completed", + output=result if isinstance(result, dict) else {"value": result}, + durationMs=_rDur, retryCount=_rRetry) + logger.info("executeGraph loop resume body node %s done (iter %d, retries=%d)", bnid, next_index, _rRetry) except PauseForHumanTaskError as e: + _updateStepLog(automation2_interface, _rStepId, "completed", + durationMs=int((time.time() - _rStepStart) * 1000)) if automation2_interface: run_ctx = dict(run.get("context") or {}) run_ctx["_loopState"] = {"loopNodeId": loop_node_id, "currentIndex": next_index, "items": items} automation2_interface.updateRun(e.runId, status="paused", nodeOutputs=dict(nodeOutputs), currentNodeId=e.nodeId, context=run_ctx) return {"success": False, "paused": True, "taskId": e.taskId, "runId": e.runId, "nodeId": e.nodeId, "nodeOutputs": dict(nodeOutputs)} + except PauseForEmailWaitError as e: + _updateStepLog(automation2_interface, _rStepId, "completed", + durationMs=int((time.time() - _rStepStart) * 1000)) + raise except Exception as ex: + _updateStepLog(automation2_interface, _rStepId, "failed", + error=str(ex), durationMs=int((time.time() - _rStepStart) * 1000)) logger.exception("executeGraph loop body node %s FAILED: %s", bnid, ex) nodeOutputs[bnid] = {"error": str(ex), "success": False} if runId and automation2_interface: @@ -237,6 +330,9 @@ async def executeGraph( nodeType = node.get("type", "") if not _is_node_on_active_path(nodeId, connectionMap, nodeOutputs): logger.info("executeGraph step %d/%d: nodeId=%s SKIP (inactive branch)", i + 1, len(ordered), nodeId) + _skipStepId = _createStepLog(automation2_interface, runId, nodeId, nodeType, status="skipped") + if _skipStepId: + _updateStepLog(automation2_interface, _skipStepId, "skipped") continue executor = _getExecutor(nodeType, services, automation2_interface) logger.info( @@ -251,8 +347,11 @@ async def executeGraph( nodeOutputs[nodeId] = None logger.debug("executeGraph node %s: no executor, output=None", nodeId) continue + _stepStartMs = time.time() + _stepId = None try: if nodeType == "flow.loop": + _stepId = _createStepLog(automation2_interface, runId, nodeId, nodeType, "running") result = await executor.execute(node, context) items = result.get("items") or [] body_ids = getLoopBodyNodeIds(nodeId, connectionMap) @@ -272,35 +371,66 @@ async def executeGraph( if not bexec: nodeOutputs[bnid] = None continue + _bStepStart = time.time() + _bStepId = _createStepLog(automation2_interface, runId, bnid, body_node.get("type", ""), "running") try: - bres = await bexec.execute(body_node, context) + bres, _bRetry = await _executeWithRetry(bexec, body_node, context) nodeOutputs[bnid] = bres - logger.info("executeGraph loop body node %s done (iter %d)", bnid, idx) + _bDur = int((time.time() - _bStepStart) * 1000) + _updateStepLog(automation2_interface, _bStepId, "completed", + output=bres if isinstance(bres, dict) else {"value": bres}, + durationMs=_bDur, retryCount=_bRetry) + logger.info("executeGraph loop body node %s done (iter %d, retries=%d)", bnid, idx, _bRetry) except PauseForHumanTaskError as e: + _updateStepLog(automation2_interface, _bStepId, "completed", + durationMs=int((time.time() - _bStepStart) * 1000)) if runId and automation2_interface: run = automation2_interface.getRun(runId) or {} run_ctx = dict(run.get("context") or {}) run_ctx["_loopState"] = {"loopNodeId": nodeId, "currentIndex": idx, "items": items} automation2_interface.updateRun(e.runId, status="paused", nodeOutputs=dict(nodeOutputs), currentNodeId=e.nodeId, context=run_ctx) return {"success": False, "paused": True, "taskId": e.taskId, "runId": e.runId, "nodeId": e.nodeId, "nodeOutputs": dict(nodeOutputs)} + except PauseForEmailWaitError as e: + _updateStepLog(automation2_interface, _bStepId, "completed", + durationMs=int((time.time() - _bStepStart) * 1000)) + raise except Exception as ex: + _updateStepLog(automation2_interface, _bStepId, "failed", + error=str(ex), durationMs=int((time.time() - _bStepStart) * 1000)) logger.exception("executeGraph loop body node %s FAILED: %s", bnid, ex) nodeOutputs[bnid] = {"error": str(ex), "success": False} if runId and automation2_interface: automation2_interface.updateRun(runId, status="failed", nodeOutputs=nodeOutputs) return {"success": False, "error": str(ex), "nodeOutputs": nodeOutputs, "failedNode": bnid} nodeOutputs[nodeId] = {"items": items, "count": len(items)} + _updateStepLog(automation2_interface, _stepId, "completed", + output={"items": len(items)}, durationMs=int((time.time() - _stepStartMs) * 1000)) logger.info("executeGraph flow.loop done: %d iterations", len(items)) else: - result = await executor.execute(node, context) + _stepStartMs = time.time() + _inputSnap = {} + for src, _, _ in connectionMap.get(nodeId, []): + if src in nodeOutputs: + _inputSnap[src] = nodeOutputs[src] + _stepId = _createStepLog(automation2_interface, runId, nodeId, nodeType, "running", _inputSnap) + result, retryCount = await _executeWithRetry(executor, node, context) nodeOutputs[nodeId] = result + _durMs = int((time.time() - _stepStartMs) * 1000) + _tokens = result.get("tokensUsed", 0) if isinstance(result, dict) else 0 + _updateStepLog(automation2_interface, _stepId, "completed", + output=result if isinstance(result, dict) else {"value": result}, + durationMs=_durMs, tokensUsed=_tokens, retryCount=retryCount) logger.info( - "executeGraph node %s done: result_type=%s result_keys=%s", + "executeGraph node %s done: result_type=%s result_keys=%s retries=%d duration=%dms", nodeId, type(result).__name__, list(result.keys()) if isinstance(result, dict) else "n/a", + retryCount, + _durMs, ) except PauseForHumanTaskError as e: + _updateStepLog(automation2_interface, _stepId, "completed", + durationMs=int((time.time() - _stepStartMs) * 1000)) logger.info("executeGraph paused for human task %s", e.taskId) return { "success": False, @@ -311,11 +441,13 @@ async def executeGraph( "nodeOutputs": dict(nodeOutputs), } except PauseForEmailWaitError as e: + _updateStepLog(automation2_interface, _stepId, "completed", + durationMs=int((time.time() - _stepStartMs) * 1000)) logger.info("executeGraph paused for email wait (run %s, node %s)", e.runId, e.nodeId) # Start email poller on-demand (only runs while workflows wait for email) try: from modules.interfaces.interfaceDbApp import getRootInterface - from modules.features.automation2.emailPoller import ensureRunning + from modules.features.graphicalEditor.emailPoller import ensureRunning root = getRootInterface() event_user = root.getUserByUsername("event") if root else None if event_user: @@ -353,8 +485,25 @@ async def executeGraph( except Exception as e: logger.exception("executeGraph node %s (%s) FAILED: %s", nodeId, nodeType, e) nodeOutputs[nodeId] = {"error": str(e), "success": False} + _durMs = int((time.time() - _stepStartMs) * 1000) + _updateStepLog(automation2_interface, _stepId, "failed", error=str(e), durationMs=_durMs) if runId and automation2_interface: automation2_interface.updateRun(runId, status="failed", nodeOutputs=nodeOutputs) + try: + _wfObj = automation2_interface.getWorkflow(workflowId) if automation2_interface and workflowId else None + _wfDict = _wfObj if isinstance(_wfObj, dict) else ( + _wfObj.model_dump() if hasattr(_wfObj, "model_dump") else {} + ) if _wfObj else {} + _shouldNotify = _wfDict.get("notifyOnFailure", True) if _wfDict else True + if _shouldNotify: + from modules.workflows.scheduler.mainScheduler import _notifyRunFailed + _notifyRunFailed( + workflowId or "", runId or "", str(e), + mandateId=mandateId, + workflowLabel=_wfDict.get("label"), + ) + except Exception: + pass return { "success": False, "error": str(e), diff --git a/modules/workflows/automation2/executors/actionNodeExecutor.py b/modules/workflows/automation2/executors/actionNodeExecutor.py index ab19964d..f2456803 100644 --- a/modules/workflows/automation2/executors/actionNodeExecutor.py +++ b/modules/workflows/automation2/executors/actionNodeExecutor.py @@ -29,7 +29,7 @@ def _is_user_connection_id(val: Any) -> bool: def _getNodeDefinition(nodeType: str) -> Optional[Dict[str, Any]]: """Get node definition by type id for _method, _action, _paramMap.""" - from modules.features.automation2.nodeDefinitions import STATIC_NODE_TYPES + from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES for node in STATIC_NODE_TYPES: if node.get("id") == nodeType: return node @@ -534,7 +534,7 @@ class ActionNodeExecutor: node: Dict[str, Any], context: Dict[str, Any], ) -> Any: - from modules.features.automation2.nodeRegistry import getNodeTypeToMethodAction + from modules.features.graphicalEditor.nodeRegistry import getNodeTypeToMethodAction from modules.workflows.automation2.graphUtils import resolveParameterReferences from modules.workflows.processing.core.actionExecutor import ActionExecutor diff --git a/modules/workflows/automation2/subAutomation2Schedule.py b/modules/workflows/automation2/subAutomation2Schedule.py index d0fb3cd8..2e551eef 100644 --- a/modules/workflows/automation2/subAutomation2Schedule.py +++ b/modules/workflows/automation2/subAutomation2Schedule.py @@ -17,12 +17,12 @@ _main_loop = None def set_main_loop(loop) -> None: global _main_loop _main_loop = loop -from modules.features.automation2.interfaceFeatureAutomation2 import ( - getAutomation2Interface, +from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import ( + getGraphicalEditorInterface as getAutomation2Interface, getAllWorkflowsForScheduling, ) -from modules.features.automation2.mainAutomation2 import getAutomation2Services -from modules.features.automation2.entryPoints import find_invocation +from modules.features.graphicalEditor.mainGraphicalEditor import getGraphicalEditorServices as getAutomation2Services +from modules.features.graphicalEditor.entryPoints import find_invocation from modules.workflows.automation2.scheduleCron import parse_cron_to_kwargs diff --git a/modules/workflows/scheduler/__init__.py b/modules/workflows/scheduler/__init__.py new file mode 100644 index 00000000..e2b0f5de --- /dev/null +++ b/modules/workflows/scheduler/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2025 Patrick Motsch +# Workflow Scheduler - consolidated scheduler with v1 incremental sync patterns diff --git a/modules/workflows/scheduler/mainScheduler.py b/modules/workflows/scheduler/mainScheduler.py new file mode 100644 index 00000000..ec156fd2 --- /dev/null +++ b/modules/workflows/scheduler/mainScheduler.py @@ -0,0 +1,459 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Consolidated Workflow Scheduler. +Replaces subAutomation2Schedule with v1-style incremental sync patterns: +- eventId tracking on AutoWorkflow for change detection +- replaceExisting=True for idempotent re-registration +- active check before execution +- Capped execution log +""" + +import asyncio +import logging +from typing import Any, Dict + +from modules.shared.eventManagement import eventManager + +logger = logging.getLogger(__name__) + +_main_loop = None + +JOB_ID_PREFIX = "graphicalEditor." +_CALLBACK_NAME = "graphicalEditor.workflow.changed" + + +def _setMainLoop(loop) -> None: + global _main_loop + _main_loop = loop + + +class WorkflowScheduler: + """Consolidated scheduler with v1 incremental sync patterns.""" + + def __init__(self): + self._eventUser = None + self._registered: Dict[str, str] = {} + + def start(self, eventUser) -> bool: + """Start scheduler: sync workflows, register callback for changes.""" + if not eventUser: + logger.warning("WorkflowScheduler: No event user provided, skipping") + return False + + self._eventUser = eventUser + + try: + eventManager.start() + self._syncScheduledWorkflows() + logger.info("WorkflowScheduler: initial sync complete") + + self._delayedSync() + + from modules.shared.callbackRegistry import callbackRegistry + callbackRegistry.register(_CALLBACK_NAME, self._onWorkflowChanged) + logger.info("WorkflowScheduler: callback registered for %s", _CALLBACK_NAME) + except Exception as e: + logger.error("WorkflowScheduler: Failed to start: %s", e) + return False + + return True + + def stop(self) -> bool: + """Remove all scheduled workflow jobs.""" + try: + self._removeAllJobs() + logger.info("WorkflowScheduler: all jobs removed") + except Exception as e: + logger.warning("WorkflowScheduler: error during stop: %s", e) + return True + + def _syncScheduledWorkflows(self) -> Dict[str, Any]: + """ + Incremental sync: only re-register jobs whose eventId has changed. + Uses AutoWorkflow.eventId for change detection (v1 pattern). + """ + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getAllWorkflowsForScheduling + from modules.workflows.automation2.scheduleCron import parse_cron_to_kwargs + + items = getAllWorkflowsForScheduling() + logger.info("WorkflowScheduler: found %d workflow(s) with trigger.schedule+cron", len(items)) + + newRegistered: Dict[str, str] = {} + activeWorkflowIds = set() + + for item in items: + workflowId = item.get("workflowId") + if not workflowId: + continue + + activeWorkflowIds.add(workflowId) + cron = item.get("cron") + mandateId = item.get("mandateId") + instanceId = item.get("featureInstanceId") + + if not instanceId or not cron: + continue + + jobId = f"{JOB_ID_PREFIX}{workflowId}" + entryPointId = item.get("entryPointId") + workflow = item.get("workflow") or {} + + asyncHandler = self._createHandler( + workflowId=workflowId, + mandateId=mandateId, + instanceId=instanceId, + entryPointId=entryPointId, + workflow=workflow, + ) + + def _makeSyncWrapper(handler): + def syncWrapper(): + loop = _main_loop + if loop and loop.is_running(): + loop.call_soon_threadsafe( + lambda: asyncio.ensure_future(handler(), loop=loop) + ) + else: + try: + asyncio.run(handler()) + except RuntimeError: + logger.warning("WorkflowScheduler: could not run handler, no event loop") + return syncWrapper + + syncWrapper = _makeSyncWrapper(asyncHandler) + + intervalSeconds = _cronToIntervalSeconds(cron) + if intervalSeconds is not None: + eventManager.registerInterval( + jobId=jobId, + func=syncWrapper, + seconds=intervalSeconds, + replaceExisting=True, + ) + else: + try: + cronKwargs = parse_cron_to_kwargs(cron) + eventManager.registerCron( + jobId=jobId, + func=syncWrapper, + cronKwargs=cronKwargs, + replaceExisting=True, + ) + except ValueError as e: + logger.warning("Workflow %s: invalid cron %r: %s", workflowId, cron, e) + continue + + newRegistered[workflowId] = jobId + mode = "interval" if intervalSeconds is not None else "cron" + logger.info( + "WorkflowScheduler: registered %s for workflow %s (%s=%s)", + jobId, workflowId, mode, + intervalSeconds if intervalSeconds is not None else cron, + ) + + self._updateEventId(workflow, workflowId, jobId) + + staleIds = set(self._registered.keys()) - activeWorkflowIds + for wfId in staleIds: + oldJobId = self._registered[wfId] + try: + eventManager.remove(oldJobId) + logger.info("WorkflowScheduler: removed stale job %s", oldJobId) + except Exception: + pass + + self._registered = newRegistered + return {"synced": len(newRegistered), "workflowsFound": len(items)} + + def _updateEventId(self, workflow: Dict, workflowId: str, jobId: str) -> None: + """Update AutoWorkflow.eventId for incremental sync tracking (v1 pattern).""" + currentEventId = workflow.get("eventId") + if currentEventId != jobId: + try: + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + from modules.interfaces.interfaceDbApp import getRootInterface + root = getRootInterface() + eventUser = root.getUserByUsername("event") if root else self._eventUser + if not eventUser: + return + mandateId = workflow.get("mandateId", "") + instanceId = workflow.get("featureInstanceId", "") + iface = getGraphicalEditorInterface(eventUser, mandateId, instanceId) + iface.updateWorkflow(workflowId, {"eventId": jobId}) + except Exception as e: + logger.debug("WorkflowScheduler: could not update eventId for %s: %s", workflowId, e) + + def _createHandler( + self, + workflowId: str, + mandateId: str, + instanceId: str, + entryPointId: str, + workflow: Dict[str, Any], + ): + """Create async handler for scheduled workflow execution with active-check.""" + eventUser = self._eventUser + + async def handler(): + logger.info("WorkflowScheduler: CRON FIRED for workflow %s", workflowId) + try: + if not eventUser: + logger.error("WorkflowScheduler: event user not available") + return + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + from modules.features.graphicalEditor.mainGraphicalEditor import getGraphicalEditorServices + from modules.workflows.automation2.executionEngine import executeGraph + from modules.workflows.processing.shared.methodDiscovery import discoverMethods + from modules.features.graphicalEditor.entryPoints import find_invocation + from modules.workflows.automation2.runEnvelope import default_run_envelope, normalize_run_envelope + + iface = getGraphicalEditorInterface(eventUser, mandateId, instanceId) + wf = iface.getWorkflow(workflowId) + if not wf or not wf.get("graph"): + logger.warning("WorkflowScheduler: workflow %s not found or no graph", workflowId) + return + if not wf.get("active", True): + logger.info("WorkflowScheduler: workflow %s inactive, skipping", workflowId) + return + + inv = find_invocation(wf, entryPointId) + if inv and (inv.get("kind") != "schedule" or not inv.get("enabled", True)): + logger.info("WorkflowScheduler: entry point %s disabled for workflow %s", entryPointId, workflowId) + return + + services = getGraphicalEditorServices( + eventUser, + mandateId=mandateId, + featureInstanceId=instanceId, + ) + discoverMethods(services) + + title = (inv or {}).get("title") or {} + label = "" + if isinstance(title, dict): + label = title.get("en") or title.get("de") or "" + elif isinstance(title, str): + label = title + + runEnv = default_run_envelope( + "schedule", + entry_point_id=entryPointId, + entry_point_label=label or None, + ) + runEnv = normalize_run_envelope(runEnv, user_id=str(eventUser.id) if eventUser else None) + + result = await executeGraph( + graph=wf["graph"], + services=services, + workflowId=workflowId, + instanceId=instanceId, + userId=None, + mandateId=mandateId, + automation2_interface=iface, + run_envelope=runEnv, + ) + logger.info( + "WorkflowScheduler: executed workflow %s success=%s paused=%s", + workflowId, result.get("success"), result.get("paused"), + ) + except Exception as e: + logger.exception("WorkflowScheduler: failed to execute workflow %s: %s", workflowId, e) + + return handler + + def _delayedSync(self) -> None: + """Delayed sync (5s) in case DB was not ready at startup.""" + import threading + + eventUser = self._eventUser + + def _run(): + import time + time.sleep(5) + try: + self._syncScheduledWorkflows() + logger.info("WorkflowScheduler: delayed sync done") + except Exception as e: + logger.warning("WorkflowScheduler: delayed sync failed: %s", e) + + t = threading.Thread(target=_run, daemon=True) + t.start() + + def _onWorkflowChanged(self, _context=None) -> None: + """Callback when a workflow is created/updated/deleted.""" + try: + self._syncScheduledWorkflows() + logger.debug("WorkflowScheduler: re-synced after workflow change") + except Exception as e: + logger.warning("WorkflowScheduler: re-sync failed: %s", e) + + def _removeAllJobs(self) -> None: + """Remove all registered workflow schedule jobs.""" + if not eventManager.scheduler: + return + for job in list(eventManager.scheduler.get_jobs()): + jid = job.id if hasattr(job, "id") else str(job) + if jid.startswith(JOB_ID_PREFIX): + try: + eventManager.remove(jid) + except Exception as e: + logger.debug("Could not remove job %s: %s", jid, e) + + +def _cronToIntervalSeconds(cron: str): + """If cron represents a simple interval, return seconds. Otherwise None.""" + if not cron or not isinstance(cron, str): + return None + parts = cron.strip().split() + if len(parts) == 5: + minute, hour, day, month, dow = parts + second = "0" + elif len(parts) == 6: + second, minute, hour, day, month, dow = parts + else: + return None + if minute.startswith("*/") and hour == "*" and day == "*" and month == "*" and dow == "*": + n = int(minute[2:]) if minute[2:].isdigit() else 0 + if n > 0: + return n * 60 + if minute == "*" and hour == "*" and day == "*" and month == "*" and dow == "*" and second == "0": + return 60 + if minute == "0" and hour.startswith("*/") and day == "*" and month == "*" and dow == "*": + n = int(hour[2:]) if hour[2:].isdigit() else 0 + if n > 0: + return n * 3600 + if len(parts) == 6 and second.startswith("*/") and minute == "*" and hour == "*" and day == "*" and month == "*" and dow in ("*", "?"): + n = int(second[2:]) if second[2:].isdigit() else 0 + if n > 0: + return n + return None + + +def _notifyRunFailed(workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None) -> None: + """Notify on workflow run failure: emit event, create in-app notification, trigger email subscription.""" + try: + eventManager.emit("graphicalEditor.run.failed", { + "workflowId": workflowId, + "runId": runId, + "error": error, + "mandateId": mandateId, + }) + logger.info("Emitted run.failed event for run %s (workflow %s)", runId, workflowId) + except Exception as e: + logger.warning("Failed to emit run.failed event: %s", e) + + _createRunFailedNotification(workflowId, runId, error, mandateId, workflowLabel) + _triggerRunFailedSubscription(workflowId, runId, error, mandateId, workflowLabel) + + +def _createRunFailedNotification( + workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None +) -> None: + """Create in-app notification for the workflow creator.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelNotification import UserNotification, NotificationType, NotificationStatus + + rootInterface = getRootInterface() + if not rootInterface: + return + + from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import getGraphicalEditorInterface + eventUser = rootInterface.getUserByUsername("event") + if not eventUser: + return + + iface = getGraphicalEditorInterface(eventUser, mandateId or "", "") + wf = iface.getWorkflow(workflowId) + if not wf: + return + + creatorId = wf.get("sysCreatedBy") if isinstance(wf, dict) else getattr(wf, "sysCreatedBy", None) + if not creatorId: + return + + label = workflowLabel or (wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", "")) + notification = UserNotification( + userId=creatorId, + type=NotificationType.SYSTEM, + status=NotificationStatus.UNREAD, + title="Workflow fehlgeschlagen", + message=f"Workflow '{label or workflowId}' ist fehlgeschlagen: {error[:200]}", + referenceType="AutoRun", + referenceId=runId, + icon="alert-triangle", + ) + rootInterface.db.recordCreate( + model_class=UserNotification, + record=notification.model_dump(), + ) + logger.info("Created in-app notification for user %s (run %s)", creatorId, runId) + except Exception as e: + logger.warning("Failed to create in-app run.failed notification: %s", e) + + +def _triggerRunFailedSubscription( + workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None +) -> None: + """Trigger the messaging subscription for run failures (email notifications).""" + try: + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelMessaging import MessagingEventParameters + + rootInterface = getRootInterface() + if not rootInterface: + return + eventUser = rootInterface.getUserByUsername("event") + if not eventUser: + return + + ctx = ServiceCenterContext( + user=eventUser, + mandate_id=mandateId or "", + feature_instance_id="", + feature_code="graphicalEditor", + ) + messagingService = getService("messaging", ctx) + + subscriptionId = "GraphicalEditorRunFailed" + eventParams = MessagingEventParameters(triggerData={ + "workflowId": workflowId, + "workflowLabel": workflowLabel or workflowId, + "runId": runId, + "error": error, + "mandateId": mandateId or "", + }) + result = messagingService.executeSubscription(subscriptionId, eventParams) + logger.info( + "Triggered run.failed subscription: sent=%d success=%s", + result.messagesSent, result.success, + ) + except FileNotFoundError: + logger.debug("Subscription function GraphicalEditorRunFailed not found (not yet registered)") + except ValueError as e: + logger.debug("Subscription GraphicalEditorRunFailed: %s", e) + except Exception as e: + logger.warning("Failed to trigger run.failed subscription: %s", e) + + +# Module-level singleton +_scheduler = WorkflowScheduler() + + +def start(eventUser) -> bool: + """Start the consolidated workflow scheduler.""" + return _scheduler.start(eventUser) + + +def stop() -> bool: + """Stop the consolidated workflow scheduler.""" + return _scheduler.stop() + + +def setMainLoop(loop) -> None: + """Set the main event loop for thread-bridge.""" + _setMainLoop(loop) diff --git a/scripts/migrate_async_to_sync.py b/scripts/migrate_async_to_sync.py index 4e18b3ea..433fd513 100644 --- a/scripts/migrate_async_to_sync.py +++ b/scripts/migrate_async_to_sync.py @@ -39,9 +39,6 @@ AUTH_DIR = GATEWAY_DIR / "modules" / "auth" # Value: set of function names that must remain async def _MUST_STAY_ASYNC: Dict[str, Set[str]] = { # --- routes/ --- - "modules/routes/routeAdminAutomationEvents.py": { - "sync_all_automation_events", # await syncAutomationEvents(...) - }, "modules/routes/routeAdminRbacExport.py": { "import_global_rbac", # await file.read() "import_mandate_rbac", # await file.read() @@ -68,9 +65,6 @@ _MUST_STAY_ASYNC: Dict[str, Set[str]] = { "refresh_token", # await request.json() }, # --- features/ --- - "modules/features/automation/routeFeatureAutomation.py": { - "execute_automation_route", # await executeAutomation(...) - }, "modules/features/chatbot/routeFeatureChatbot.py": { "stream_chatbot_start", # await chatProcess(...), contains async event_stream generator "event_stream", # await request.is_disconnected(), await asyncio.wait_for(...) diff --git a/scripts/script_db_export_migration.py b/scripts/script_db_export_migration.py index 73c13b25..f286327a 100644 --- a/scripts/script_db_export_migration.py +++ b/scripts/script_db_export_migration.py @@ -102,8 +102,6 @@ except Exception as e: # Alle PowerOn Datenbanken (für Export / Migration-Skripte) ALL_DATABASES = [ "poweron_app", - "poweron_automation", - "poweron_automation2", "poweron_billing", "poweron_chat", "poweron_chatbot", @@ -128,8 +126,6 @@ DATABASE_CONFIG = { "poweron_management": "DB_MANAGEMENT", "poweron_realestate": "DB_REALESTATE", "poweron_trustee": "DB_TRUSTEE", - "poweron_automation": "DB", - "poweron_automation2": "DB", "poweron_billing": "DB", "poweron_commcoach": "DB", "poweron_knowledge": "DB", diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py deleted file mode 100644 index 5ca9be17..00000000 --- a/tests/functional/test05_workflow_with_documents.py +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Workflow Test with Documents - Tests chat workflow execution with uploaded documents -Simulates the UI route flow: upload files, start workflow with prompt and documents -""" - -import asyncio -import json -import sys -import os -import time -from typing import Dict, Any, List, Optional - -# Add the gateway to path (go up 2 levels from tests/functional/) -_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _gateway_path not in sys.path: - sys.path.insert(0, _gateway_path) - -# Import the service initialization -from modules.serviceHub import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User -from modules.workflows.automation import chatStart -import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat - - -class WorkflowWithDocumentsTester: - def __init__(self): - # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelUam import Mandate - rootInterface = getRootInterface() - self.testUser = rootInterface.currentUser - # Get initial mandate ID for testing (User has no mandateId - use initial mandate) - self.testMandateId = rootInterface.getInitialId(Mandate) - - # Initialize services using the existing system - self.services = getServices(self.testUser, None) # Test user, no workflow - self.workflow = None - self.testResults = {} - - async def initialize(self): - """Initialize the test environment.""" - # Set logging level to INFO to see workflow progress - import logging - logging.getLogger().setLevel(logging.INFO) - - print(f"Initialized test with user: {self.testUser.id}") - print(f"Test Mandate ID: {self.testMandateId}") - - def createCsvTemplate(self) -> str: - """Create a CSV template file for prime numbers.""" - csvContent = """Primzahl,Index -2,1 -3,2 -5,3 -7,4 -11,5 -13,6 -17,7 -19,8 -23,9 -29,10 -""" - return csvContent - - def createSecondDocument(self) -> str: - """Create a second text document with instructions.""" - docContent = """Anweisungen zur Primzahlgenerierung: - -1. Generiere Primzahlen -2. Formatiere sie in einer Tabelle mit 10 Spalten pro Zeile -3. Verwende das bereitgestellte CSV-Vorlagenformat -4. Stelle sicher, dass alle Zahlen korrekt formatiert sind -5. Füge eine Index-Spalte hinzu, die bei 1 beginnt - - -""" - return docContent - - async def uploadFiles(self) -> List[str]: - """Upload test files to the filesystem and return their file IDs.""" - print("\n" + "="*60) - print("UPLOADING TEST FILES") - print("="*60) - - fileIds = [] - - # Create CSV template file - csvContent = self.createCsvTemplate() - csvFileName = "prime_numbers_template.csv" - - print(f"Creating CSV template: {csvFileName}") - print(f"Content length: {len(csvContent)} bytes") - - # Create file in component storage - csvFileItem = self.services.interfaceDbComponent.createFile( - name=csvFileName, - mimeType="text/csv", - content=csvContent.encode('utf-8') - ) - # Persist file data - self.services.interfaceDbComponent.createFileData(csvFileItem.id, csvContent.encode('utf-8')) - - fileIds.append(csvFileItem.id) - print(f"✅ Created CSV file with ID: {csvFileItem.id}") - print(f" File name: {csvFileItem.fileName}") - print(f" MIME type: {csvFileItem.mimeType}") - - # Create second text document - docContent = self.createSecondDocument() - docFileName = "prime_numbers_instructions.txt" - - print(f"\nCreating instruction document: {docFileName}") - print(f"Content length: {len(docContent)} bytes") - - # Create file in component storage - docFileItem = self.services.interfaceDbComponent.createFile( - name=docFileName, - mimeType="text/plain", - content=docContent.encode('utf-8') - ) - # Persist file data - self.services.interfaceDbComponent.createFileData(docFileItem.id, docContent.encode('utf-8')) - - fileIds.append(docFileItem.id) - print(f"✅ Created instruction file with ID: {docFileItem.id}") - print(f" File name: {docFileItem.fileName}") - print(f" MIME type: {docFileItem.mimeType}") - - return fileIds - - async def startWorkflow(self, prompt: str, fileIds: List[str]) -> None: - """Start a chat workflow with prompt and documents.""" - print("\n" + "="*60) - print("STARTING WORKFLOW") - print("="*60) - - print(f"Prompt: {prompt}") - print(f"Number of files: {len(fileIds)}") - print(f"File IDs: {fileIds}") - - # Create UserInputRequest - userInput = UserInputRequest( - prompt=prompt, - listFileId=fileIds, - userLanguage="en" - ) - - # Start workflow (this is async and returns immediately) - print("\nCalling chatStart...") - self.workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - print(f"✅ Workflow started with ID: {self.workflow.id}") - print(f" Status: {self.workflow.status}") - print(f" Mode: {self.workflow.workflowMode}") - print(f" Current Round: {self.workflow.currentRound}") - - async def waitForWorkflowCompletion(self, maxWaitTime: Optional[int] = None) -> bool: - """Wait for workflow to complete, checking status periodically. - - Args: - maxWaitTime: Maximum wait time in seconds. If None, wait indefinitely. - """ - print("\n" + "="*60) - print("WAITING FOR WORKFLOW COMPLETION") - if maxWaitTime: - print(f"Maximum wait time: {maxWaitTime} seconds") - else: - print("Waiting indefinitely (no timeout)") - print("="*60) - - if not self.workflow: - print("❌ No workflow to wait for") - return False - - startTime = time.time() - checkInterval = 2 # Check every 2 seconds - lastStatus = None - - while True: - # Check timeout if maxWaitTime is set - if maxWaitTime is not None: - elapsed = time.time() - startTime - if elapsed >= maxWaitTime: - print(f"\n⚠️ Workflow did not complete within {maxWaitTime} seconds") - print(f" Final status: {self.workflow.status}") - return False - - # Get current workflow status - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) - - if not currentWorkflow: - print("❌ Workflow not found in database") - return False - - currentStatus = currentWorkflow.status - elapsed = int(time.time() - startTime) - - # Print status if it changed - if currentStatus != lastStatus: - print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)") - lastStatus = currentStatus - - # Check if workflow is complete - if currentStatus in ["completed", "stopped", "failed"]: - self.workflow = currentWorkflow - print(f"\n✅ Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)") - return currentStatus == "completed" - - # Wait before next check - await asyncio.sleep(checkInterval) - - def analyzeWorkflowResults(self) -> Dict[str, Any]: - """Analyze workflow results and extract information.""" - print("\n" + "="*60) - print("ANALYZING WORKFLOW RESULTS") - print("="*60) - - if not self.workflow: - return {"error": "No workflow to analyze"} - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - workflow = interfaceDbChat.getWorkflow(self.workflow.id) - - if not workflow: - return {"error": "Workflow not found"} - - # Get unified chat data - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - - # Count messages - messages = chatData.get("messages", []) - userMessages = [m for m in messages if m.get("role") == "user"] - assistantMessages = [m for m in messages if m.get("role") == "assistant"] - - # Count documents - documents = chatData.get("documents", []) - - # Get logs - logs = chatData.get("logs", []) - - # Get stats - stats = chatData.get("stats", []) - - results = { - "workflowId": workflow.id, - "status": workflow.status, - "workflowMode": str(workflow.workflowMode) if hasattr(workflow, 'workflowMode') else None, - "currentRound": workflow.currentRound, - "totalTasks": workflow.totalTasks, - "totalActions": workflow.totalActions, - "messageCount": len(messages), - "userMessageCount": len(userMessages), - "assistantMessageCount": len(assistantMessages), - "documentCount": len(documents), - "logCount": len(logs), - "statCount": len(stats), - "messages": messages, - "documents": documents, - "logs": logs, - "stats": stats - } - - print(f"Workflow ID: {results['workflowId']}") - print(f"Status: {results['status']}") - print(f"Mode: {results['workflowMode']}") - print(f"Round: {results['currentRound']}") - print(f"Tasks: {results['totalTasks']}") - print(f"Actions: {results['totalActions']}") - print(f"Messages: {results['messageCount']} (User: {results['userMessageCount']}, Assistant: {results['assistantMessageCount']})") - print(f"Documents: {results['documentCount']}") - print(f"Logs: {results['logCount']}") - print(f"Stats: {results['statCount']}") - - # Print first user message - if userMessages: - print(f"\nFirst user message:") - print(f" {userMessages[0].get('message', '')[:200]}...") - - # Print last assistant message - if assistantMessages: - print(f"\nLast assistant message:") - lastMsg = assistantMessages[-1] - print(f" {lastMsg.get('message', '')[:200]}...") - if lastMsg.get('documents'): - print(f" Documents attached: {len(lastMsg['documents'])}") - - # Print document names - if documents: - print(f"\nGenerated documents:") - for doc in documents: - print(f" - {doc.get('fileName', 'unknown')} ({doc.get('fileSize', 0)} bytes)") - - return results - - async def runTest(self): - """Run the complete test.""" - print("\n" + "="*80) - print("WORKFLOW TEST WITH DOCUMENTS") - print("="*80) - - try: - # Initialize - await self.initialize() - - # Upload files - fileIds = await self.uploadFiles() - - # Start workflow with prompt and files - prompt = "Generiere die ersten 4000 Primzahlen in einer Tabelle mit 10 Spalten pro Zeile." - await self.startWorkflow(prompt, fileIds) - - # Wait for completion (no timeout - wait indefinitely) - completed = await self.waitForWorkflowCompletion() - - # Analyze results - results = self.analyzeWorkflowResults() - - self.testResults = { - "completed": completed, - "results": results - } - - print("\n" + "="*80) - print("TEST SUMMARY") - print("="*80) - print(f"Workflow completed: {'✅' if completed else '❌'}") - print(f"Status: {results.get('status', 'unknown')}") - print(f"Messages: {results.get('messageCount', 0)}") - print(f"Documents: {results.get('documentCount', 0)}") - - return self.testResults - - except Exception as e: - import traceback - print(f"\n❌ Test failed with error: {type(e).__name__}: {str(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - self.testResults = { - "completed": False, - "error": str(e), - "traceback": traceback.format_exc() - } - return self.testResults - - -async def main(): - """Run workflow test with documents.""" - tester = WorkflowWithDocumentsTester() - results = await tester.runTest() - - # Print final results as JSON for easy parsing - print("\n" + "="*80) - print("FINAL RESULTS (JSON)") - print("="*80) - print(json.dumps(results, indent=2, default=str)) - - -if __name__ == "__main__": - asyncio.run(main()) - diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py deleted file mode 100644 index 91f1cccc..00000000 --- a/tests/functional/test06_workflow_prompt_variations.py +++ /dev/null @@ -1,471 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Workflow Test with Prompt Variations - Tests different workflow scenarios: -1. Simple prompt for short answer (no documents) -2. Merge 2 documents and output as Word document -3. Structured data output as Excel file -""" - -import asyncio -import json -import sys -import os -import time -from typing import Dict, Any, List, Optional - -# Add the gateway to path (go up 2 levels from tests/functional/) -_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _gateway_path not in sys.path: - sys.path.insert(0, _gateway_path) - -# Import the service initialization -from modules.serviceHub import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User -from modules.workflows.automation import chatStart -import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat - - -class WorkflowPromptVariationsTester: - def __init__(self): - # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelUam import Mandate - rootInterface = getRootInterface() - self.testUser = rootInterface.currentUser - # Get initial mandate ID for testing (User has no mandateId - use initial mandate) - self.testMandateId = rootInterface.getInitialId(Mandate) - - # Initialize services using the existing system - self.services = getServices(self.testUser, None) # Test user, no workflow - self.testResults = {} - - async def initialize(self): - """Initialize the test environment.""" - # Set logging level to INFO to see workflow progress - import logging - logging.getLogger().setLevel(logging.INFO) - - print(f"Initialized test with user: {self.testUser.id}") - print(f"Test Mandate ID: {self.testMandateId}") - - def _createFile(self, fileName: str, mimeType: str, content: str) -> str: - """Helper method to create a file and return its ID.""" - fileItem = self.services.interfaceDbComponent.createFile( - name=fileName, - mimeType=mimeType, - content=content.encode('utf-8') - ) - self.services.interfaceDbComponent.createFileData(fileItem.id, content.encode('utf-8')) - return fileItem.id - - async def _startWorkflow(self, prompt: str, fileIds: List[str] = None) -> Any: - """Start a chat workflow with prompt and optional documents.""" - if fileIds is None: - fileIds = [] - - print(f"\nPrompt: {prompt}") - print(f"Number of files: {len(fileIds)}") - if fileIds: - print(f"File IDs: {fileIds}") - - # Create UserInputRequest - userInput = UserInputRequest( - prompt=prompt, - listFileId=fileIds, - userLanguage="en" - ) - - # Start workflow (this is async and returns immediately) - workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - print(f"✅ Workflow started with ID: {workflow.id}") - print(f" Status: {workflow.status}") - print(f" Mode: {workflow.workflowMode}") - - return workflow - - async def _waitForWorkflowCompletion(self, workflow: Any, maxWaitTime: Optional[int] = None) -> bool: - """Wait for workflow to complete, checking status periodically. - - Args: - workflow: The workflow object to wait for - maxWaitTime: Maximum wait time in seconds. If None, wait indefinitely. - """ - if maxWaitTime: - print(f"Maximum wait time: {maxWaitTime} seconds") - else: - print("Waiting indefinitely (no timeout)") - - startTime = time.time() - checkInterval = 2 # Check every 2 seconds - lastStatus = None - - while True: - # Check timeout if maxWaitTime is set - if maxWaitTime is not None: - elapsed = time.time() - startTime - if elapsed >= maxWaitTime: - print(f"\n⚠️ Workflow did not complete within {maxWaitTime} seconds") - print(f" Final status: {workflow.status}") - return False - - # Get current workflow status - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - currentWorkflow = interfaceDbChat.getWorkflow(workflow.id) - - if not currentWorkflow: - print("❌ Workflow not found in database") - return False - - currentStatus = currentWorkflow.status - elapsed = int(time.time() - startTime) - - # Print status if it changed - if currentStatus != lastStatus: - print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)") - lastStatus = currentStatus - - # Check if workflow is complete - if currentStatus in ["completed", "stopped", "failed"]: - print(f"\n✅ Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)") - return currentStatus == "completed" - - # Wait before next check - await asyncio.sleep(checkInterval) - - def _analyzeWorkflowResults(self, workflow: Any) -> Dict[str, Any]: - """Analyze workflow results and extract information.""" - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - workflow = interfaceDbChat.getWorkflow(workflow.id) - - if not workflow: - return {"error": "Workflow not found"} - - # Get unified chat data - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - - # Extract messages and documents from items - items = chatData.get("items", []) - messages = [] - allDocuments = [] - - for item in items: - if item.get("type") == "message": - message = item.get("item") - if message: - # Convert ChatMessage to dict if needed - if hasattr(message, 'dict'): - msgDict = message.dict() - elif hasattr(message, '__dict__'): - msgDict = message.__dict__ - else: - msgDict = message if isinstance(message, dict) else {} - - messages.append(msgDict) - - # Extract documents from message - msgDocuments = msgDict.get("documents", []) - if msgDocuments: - for doc in msgDocuments: - # Convert ChatDocument to dict if needed - if hasattr(doc, 'dict'): - docDict = doc.dict() - elif hasattr(doc, '__dict__'): - docDict = doc.__dict__ - else: - docDict = doc if isinstance(doc, dict) else {} - - # Only add if not already in list (avoid duplicates) - docId = docDict.get("id") or docDict.get("fileId") - if docId and not any(d.get("id") == docId or d.get("fileId") == docId for d in allDocuments): - allDocuments.append(docDict) - - userMessages = [m for m in messages if m.get("role") == "user"] - assistantMessages = [m for m in messages if m.get("role") == "assistant"] - - results = { - "workflowId": workflow.id, - "status": workflow.status, - "workflowMode": str(workflow.workflowMode) if hasattr(workflow, 'workflowMode') else None, - "currentRound": workflow.currentRound, - "totalTasks": workflow.totalTasks, - "totalActions": workflow.totalActions, - "messageCount": len(messages), - "userMessageCount": len(userMessages), - "assistantMessageCount": len(assistantMessages), - "documentCount": len(allDocuments), - "documents": allDocuments - } - - print(f" Workflow ID: {results['workflowId']}") - print(f" Status: {results['status']}") - print(f" Messages: {results['messageCount']} (User: {results['userMessageCount']}, Assistant: {results['assistantMessageCount']})") - print(f" Documents: {results['documentCount']}") - - # Print document names - if allDocuments: - print(f" Generated documents:") - for doc in allDocuments: - fileName = doc.get("fileName") or doc.get("documentName") or "unknown" - fileSize = doc.get("fileSize") or doc.get("size") or 0 - print(f" - {fileName} ({fileSize} bytes)") - - return results - - async def testSimplePrompt(self) -> Dict[str, Any]: - """Test 1: Simple prompt for a short answer (no documents).""" - print("\n" + "="*80) - print("TEST 1: SIMPLE PROMPT FOR SHORT ANSWER") - print("="*80) - - try: - prompt = "What is the capital of France? Answer in one sentence." - - workflow = await self._startWorkflow(prompt, []) - completed = await self._waitForWorkflowCompletion(workflow, maxWaitTime=120) - results = self._analyzeWorkflowResults(workflow) - - return { - "testName": "Simple Prompt", - "completed": completed, - "results": results - } - except Exception as e: - import traceback - print(f"❌ Test failed: {type(e).__name__}: {str(e)}") - return { - "testName": "Simple Prompt", - "completed": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - async def testMergeDocumentsToWord(self) -> Dict[str, Any]: - """Test 2: Merge 2 documents and output as Word document.""" - print("\n" + "="*80) - print("TEST 2: MERGE 2 DOCUMENTS AND OUTPUT AS WORD") - print("="*80) - - try: - # Create first document - doc1Content = """Project Overview - -This document outlines the key objectives for our new software project. -The project aims to develop a modern web application with the following features: -- User authentication and authorization -- Real-time data synchronization -- Responsive design for mobile and desktop -- Integration with third-party APIs - -Timeline: 6 months -Budget: $500,000 -""" - - # Create second document - doc2Content = """Technical Specifications - -Architecture: -- Frontend: React with TypeScript -- Backend: Python with FastAPI -- Database: PostgreSQL -- Deployment: Docker containers on AWS - -Key Requirements: -- Support for 10,000 concurrent users -- 99.9% uptime SLA -- End-to-end encryption for sensitive data -- Comprehensive logging and monitoring - -Team Size: 8 developers, 2 designers, 1 project manager -""" - - print("\nCreating documents to merge...") - doc1Id = self._createFile("project_overview.txt", "text/plain", doc1Content) - print(f"✅ Created document 1 with ID: {doc1Id}") - - doc2Id = self._createFile("technical_specs.txt", "text/plain", doc2Content) - print(f"✅ Created document 2 with ID: {doc2Id}") - - prompt = "Merge these two documents into a single comprehensive Word document. Include both the project overview and technical specifications in a well-formatted document with proper headings and sections." - - workflow = await self._startWorkflow(prompt, [doc1Id, doc2Id]) - completed = await self._waitForWorkflowCompletion(workflow, maxWaitTime=300) - results = self._analyzeWorkflowResults(workflow) - - # Check if Word document was created - wordDocFound = False - if results.get("documents"): - for doc in results["documents"]: - fileName = doc.get("fileName", "").lower() - if fileName.endswith(".docx") or fileName.endswith(".doc"): - wordDocFound = True - print(f" ✅ Word document found: {doc.get('fileName')}") - - if not wordDocFound: - print(" ⚠️ Warning: No Word document (.docx or .doc) found in results") - - return { - "testName": "Merge Documents to Word", - "completed": completed, - "wordDocumentFound": wordDocFound, - "results": results - } - except Exception as e: - import traceback - print(f"❌ Test failed: {type(e).__name__}: {str(e)}") - return { - "testName": "Merge Documents to Word", - "completed": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - async def testStructuredDataToExcel(self) -> Dict[str, Any]: - """Test 3: Structured data output as Excel file.""" - print("\n" + "="*80) - print("TEST 3: STRUCTURED DATA OUTPUT AS EXCEL") - print("="*80) - - try: - # Create structured data as JSON - structuredData = { - "employees": [ - {"id": 1, "name": "John Doe", "department": "Engineering", "salary": 95000, "startDate": "2020-01-15"}, - {"id": 2, "name": "Jane Smith", "department": "Marketing", "salary": 85000, "startDate": "2019-03-20"}, - {"id": 3, "name": "Bob Johnson", "department": "Engineering", "salary": 100000, "startDate": "2018-06-10"}, - {"id": 4, "name": "Alice Williams", "department": "HR", "salary": 75000, "startDate": "2021-09-05"}, - {"id": 5, "name": "Charlie Brown", "department": "Sales", "salary": 80000, "startDate": "2020-11-12"}, - {"id": 6, "name": "Diana Prince", "department": "Engineering", "salary": 110000, "startDate": "2017-04-22"}, - {"id": 7, "name": "Edward Norton", "department": "Marketing", "salary": 90000, "startDate": "2019-08-30"}, - {"id": 8, "name": "Fiona Green", "department": "HR", "salary": 78000, "startDate": "2022-01-18"} - ], - "departments": [ - {"name": "Engineering", "budget": 500000, "headCount": 3}, - {"name": "Marketing", "budget": 300000, "headCount": 2}, - {"name": "HR", "budget": 200000, "headCount": 2}, - {"name": "Sales", "budget": 250000, "headCount": 1} - ] - } - - jsonContent = json.dumps(structuredData, indent=2) - - print("\nCreating structured data file...") - dataFileId = self._createFile("employee_data.json", "application/json", jsonContent) - print(f"✅ Created data file with ID: {dataFileId}") - - prompt = "Create an Excel file from this structured data. Include two sheets: one for employees with all their details, and one for departments with summary information. Format the data nicely with proper column headers and make it easy to read." - - workflow = await self._startWorkflow(prompt, [dataFileId]) - completed = await self._waitForWorkflowCompletion(workflow, maxWaitTime=300) - results = self._analyzeWorkflowResults(workflow) - - # Check if Excel document was created - excelDocFound = False - if results.get("documents"): - for doc in results["documents"]: - fileName = doc.get("fileName", "").lower() - if fileName.endswith(".xlsx") or fileName.endswith(".xls"): - excelDocFound = True - print(f" ✅ Excel document found: {doc.get('fileName')}") - - if not excelDocFound: - print(" ⚠️ Warning: No Excel document (.xlsx or .xls) found in results") - - return { - "testName": "Structured Data to Excel", - "completed": completed, - "excelDocumentFound": excelDocFound, - "results": results - } - except Exception as e: - import traceback - print(f"❌ Test failed: {type(e).__name__}: {str(e)}") - return { - "testName": "Structured Data to Excel", - "completed": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - async def runAllTests(self): - """Run all three test cases.""" - print("\n" + "="*80) - print("WORKFLOW PROMPT VARIATIONS TEST SUITE") - print("="*80) - - try: - # Initialize - await self.initialize() - - # Run all tests - test1Results = await self.testSimplePrompt() - test2Results = await self.testMergeDocumentsToWord() - test3Results = await self.testStructuredDataToExcel() - - self.testResults = { - "test1": test1Results, - "test2": test2Results, - "test3": test3Results, - "summary": { - "totalTests": 3, - "passedTests": sum([ - 1 if test1Results.get("completed") else 0, - 1 if test2Results.get("completed") else 0, - 1 if test3Results.get("completed") else 0 - ]), - "failedTests": sum([ - 1 if not test1Results.get("completed") else 0, - 1 if not test2Results.get("completed") else 0, - 1 if not test3Results.get("completed") else 0 - ]) - } - } - - print("\n" + "="*80) - print("TEST SUITE SUMMARY") - print("="*80) - print(f"Test 1 - Simple Prompt: {'✅ PASSED' if test1Results.get('completed') else '❌ FAILED'}") - print(f"Test 2 - Merge to Word: {'✅ PASSED' if test2Results.get('completed') else '❌ FAILED'}") - if test2Results.get('wordDocumentFound'): - print(f" Word document created: ✅") - print(f"Test 3 - Data to Excel: {'✅ PASSED' if test3Results.get('completed') else '❌ FAILED'}") - if test3Results.get('excelDocumentFound'): - print(f" Excel document created: ✅") - print(f"\nTotal: {self.testResults['summary']['passedTests']}/{self.testResults['summary']['totalTests']} tests passed") - - return self.testResults - - except Exception as e: - import traceback - print(f"\n❌ Test suite failed with error: {type(e).__name__}: {str(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - self.testResults = { - "error": str(e), - "traceback": traceback.format_exc() - } - return self.testResults - - -async def main(): - """Run workflow prompt variations test suite.""" - tester = WorkflowPromptVariationsTester() - results = await tester.runAllTests() - - # Print final results as JSON for easy parsing - print("\n" + "="*80) - print("FINAL RESULTS (JSON)") - print("="*80) - print(json.dumps(results, indent=2, default=str)) - - -if __name__ == "__main__": - asyncio.run(main()) - diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py deleted file mode 100644 index 844863a2..00000000 --- a/tests/functional/test09_document_generation_formats.py +++ /dev/null @@ -1,735 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Document Generation Formats Test - Tests document generation in all supported formats -Tests HTML, PDF, DOCX, XLSX, and PPTX generation with images and various content types. -""" - -import asyncio -import json -import sys -import os -import time -import base64 -from typing import Dict, Any, List, Optional - -# Add the gateway to path (go up 2 levels from tests/functional/) -_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _gateway_path not in sys.path: - sys.path.insert(0, _gateway_path) - -# Import the service initialization -from modules.serviceHub import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User -from modules.workflows.automation import chatStart -import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat - - -class DocumentGenerationFormatsTester: - def __init__(self): - # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelUam import Mandate - rootInterface = getRootInterface() - self.testUser = rootInterface.currentUser - # Get initial mandate ID for testing (User has no mandateId - use initial mandate) - self.testMandateId = rootInterface.getInitialId(Mandate) - - # Initialize services using the existing system - self.services = getServices(self.testUser, None) # Test user, no workflow - self.workflow = None - self.testResults = {} - self.generatedDocuments = {} - self.pdfFileId = None # Store PDF file ID for reuse - - async def initialize(self): - """Initialize the test environment.""" - # Enable debug file logging for tests - from modules.shared.configuration import APP_CONFIG - APP_CONFIG.set("APP_DEBUG_CHAT_WORKFLOW_ENABLED", True) - - # Set logging level to INFO to see workflow progress - import logging - logging.getLogger().setLevel(logging.INFO) - - print(f"Initialized test with user: {self.testUser.id}") - print(f"Test Mandate ID: {self.testMandateId}") - print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") - - # Upload PDF file for testing - await self.uploadPdfFile() - - async def uploadPdfFile(self): - """Upload the PDF file and store its file ID.""" - pdfPath = os.path.join(os.path.dirname(__file__), "..", "..", "..", "local", "temp", "B2025-02c.pdf") - pdfPath = os.path.abspath(pdfPath) - - if not os.path.exists(pdfPath): - print(f"⚠️ Warning: PDF file not found at {pdfPath}") - print(" Test will continue without PDF attachment") - return - - try: - # Read PDF file - with open(pdfPath, "rb") as f: - pdfContent = f.read() - - # Create file using services.interfaceDbComponent - if not hasattr(self.services, 'interfaceDbComponent') or not self.services.interfaceDbComponent: - print("⚠️ Warning: interfaceDbComponent not available in services") - print(" Test will continue without PDF attachment") - return - - interfaceDbComponent = self.services.interfaceDbComponent - - fileItem = interfaceDbComponent.createFile( - name="B2025-02c.pdf", - mimeType="application/pdf", - content=pdfContent - ) - - # Store file data - interfaceDbComponent.createFileData(fileItem.id, pdfContent) - - self.pdfFileId = fileItem.id - print(f"✅ Uploaded PDF file: {fileItem.fileName} (ID: {self.pdfFileId}, Size: {len(pdfContent)} bytes)") - - except Exception as e: - import traceback - print(f"⚠️ Warning: Failed to upload PDF file: {str(e)}") - print(f" Traceback: {traceback.format_exc()}") - print(" Test will continue without PDF attachment") - - def createTestPrompt(self, format: str) -> str: - """Create a unified test prompt for document generation in the specified format. - - The prompt requests: - - Extraction of images from the attached PDF - - Generation of a new image - - Document creation with both images - """ - basePrompt = ( - "Create a professional document about 'Fuel Station Receipt Analysis' with the following content:\n" - "1) A main title\n" - "2) An introduction paragraph explaining the receipt analysis\n" - "3) Extract and include the image from the attached PDF document (B2025-02c.pdf)\n" - "4) A section analyzing the receipt data with bullet points\n" - "5) Generate a new image showing a visual representation of fuel consumption trends\n" - "6) A conclusion paragraph with recommendations\n\n" - "Make sure to include both: the image extracted from the PDF and the newly generated image.\n" - f"Format the output as {format.upper()}." - ) - return basePrompt - - def createRefactoringTestPrompt(self, testType: str, format: str = "html") -> str: - """Create test prompts for specific refactoring features. - - Args: - testType: Type of refactoring test: - - "intent_analysis": Test DocumentIntent analysis - - "conditional_extraction": Test conditional extraction (extract vs render) - - "image_render": Test image rendering as asset - - "multi_document": Test multi-document rendering - - "metadata_preservation": Test metadata preservation - format: Output format (default: html) - """ - prompts = { - "intent_analysis": ( - "Create a document with the following requirements:\n" - "1) Extract text content from the attached PDF\n" - "2) Include images from the PDF as visual elements (render them, don't extract text from them)\n" - "3) Generate a summary document\n\n" - "This tests that the system correctly identifies which documents need extraction vs rendering." - ), - "conditional_extraction": ( - "Create a document that:\n" - "1) Extracts and uses text from the attached PDF\n" - "2) Renders images from the PDF as visual assets (not as extracted text)\n" - "3) Generates new content based on the extracted text\n\n" - "This tests conditional extraction - only extract what needs extraction, render what needs rendering." - ), - "image_render": ( - "Create a document that includes images from the attached PDF.\n" - "The images should be rendered as visual elements in the document, not extracted as text.\n" - "Include a title and description for each image.\n\n" - "This tests the image asset pipeline with render intent." - ), - "multi_document": ( - "Create multiple separate documents:\n" - "1) Document 1: Summary of the PDF content\n" - "2) Document 2: Analysis of the PDF content\n" - "3) Document 3: Recommendations based on the PDF content\n\n" - "Each document should be separate and complete.\n" - "This tests multi-document generation and rendering." - ), - "metadata_preservation": ( - "Create a document that extracts content from the attached PDF.\n" - "The document should clearly show which content came from which source document.\n" - "Include source references in the generated content.\n\n" - "This tests that metadata (documentId, mimeType) is preserved in the generation prompt." - ) - } - - prompt = prompts.get(testType, self.createTestPrompt(format)) - return f"{prompt}\n\nFormat the output as {format.upper()}." - - async def generateDocumentInFormat(self, format: str) -> Dict[str, Any]: - """Generate a document in the specified format using workflow.""" - print("\n" + "="*80) - print(f"GENERATING DOCUMENT IN {format.upper()} FORMAT") - print("="*80) - - prompt = self.createTestPrompt(format) - print(f"Prompt: {prompt[:200]}...") - - # Create user input request with PDF file attachment - listFileId = [] - if self.pdfFileId: - listFileId = [self.pdfFileId] - print(f"Attaching PDF file (ID: {self.pdfFileId})") - else: - print("⚠️ No PDF file attached (file upload may have failed)") - - # Create user input request - userInput = UserInputRequest( - prompt=prompt, - listFileId=listFileId, - userLanguage="en" - ) - - # Start workflow - print(f"\nStarting workflow for {format.upper()} generation...") - workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - if not workflow: - return { - "success": False, - "error": "Failed to start workflow" - } - - self.workflow = workflow - print(f"Workflow started: {workflow.id}") - - # Wait for workflow completion (no timeout - wait indefinitely) - print(f"Waiting for workflow completion...") - completed = await self.waitForWorkflowCompletion(timeout=None) - - if not completed: - return { - "success": False, - "error": "Workflow did not complete", - "workflowId": workflow.id, - "status": workflow.status if workflow else "unknown" - } - - # Analyze results - results = self.analyzeWorkflowResults() - - # Extract documents for this format - documents = results.get("documents", []) - formatDocuments = [d for d in documents if d.get("fileName", "").endswith(f".{format.lower()}")] - - return { - "success": True, - "format": format, - "workflowId": workflow.id, - "status": results.get("status"), - "documentCount": len(formatDocuments), - "documents": formatDocuments, - "results": results - } - - async def waitForWorkflowCompletion(self, timeout: Optional[int] = None, checkInterval: int = 2) -> bool: - """Wait for workflow to complete.""" - if not self.workflow: - return False - - startTime = time.time() - lastStatus = None - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - - if timeout is None: - print("Waiting indefinitely (no timeout)") - - while True: - # Check timeout only if specified - if timeout is not None and time.time() - startTime > timeout: - print(f"\n⏱️ Timeout after {timeout} seconds") - return False - - # Get current workflow status - try: - currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) - if not currentWorkflow: - print("\n❌ Workflow not found") - return False - - currentStatus = currentWorkflow.status - elapsed = int(time.time() - startTime) - - # Print status if it changed - if currentStatus != lastStatus: - print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)") - lastStatus = currentStatus - - # Check if workflow is complete - if currentStatus in ["completed", "stopped", "failed"]: - self.workflow = currentWorkflow - statusIcon = "✅" if currentStatus == "completed" else "❌" - print(f"\n{statusIcon} Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)") - return currentStatus == "completed" - - # Wait before next check - await asyncio.sleep(checkInterval) - - except Exception as e: - print(f"\n⚠️ Error checking workflow status: {str(e)}") - await asyncio.sleep(checkInterval) - - def analyzeWorkflowResults(self) -> Dict[str, Any]: - """Analyze workflow results and extract information.""" - if not self.workflow: - return {"error": "No workflow to analyze"} - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - workflow = interfaceDbChat.getWorkflow(self.workflow.id) - - if not workflow: - return {"error": "Workflow not found"} - - # Get unified chat data - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - - # Count messages - messages = chatData.get("messages", []) - userMessages = [m for m in messages if m.get("role") == "user"] - assistantMessages = [m for m in messages if m.get("role") == "assistant"] - - # Count documents - documents = chatData.get("documents", []) - - # Get logs - logs = chatData.get("logs", []) - - results = { - "workflowId": workflow.id, - "status": workflow.status, - "workflowMode": str(workflow.workflowMode) if hasattr(workflow, 'workflowMode') else None, - "currentRound": workflow.currentRound, - "totalTasks": workflow.totalTasks, - "totalActions": workflow.totalActions, - "messageCount": len(messages), - "userMessageCount": len(userMessages), - "assistantMessageCount": len(assistantMessages), - "documentCount": len(documents), - "logCount": len(logs), - "documents": documents, - "logs": logs - } - - print(f"\nWorkflow Results:") - print(f" Status: {results['status']}") - print(f" Tasks: {results['totalTasks']}") - print(f" Actions: {results['totalActions']}") - print(f" Messages: {results['messageCount']}") - print(f" Documents: {results['documentCount']}") - - # Print document details - if documents: - print(f"\nGenerated Documents:") - for doc in documents: - fileName = doc.get("fileName", "unknown") - fileSize = doc.get("fileSize", 0) - mimeType = doc.get("mimeType", "unknown") - print(f" - {fileName} ({fileSize} bytes, {mimeType})") - - return results - - def verifyDocumentFormat(self, document: Dict[str, Any], expectedFormat: str) -> Dict[str, Any]: - """Verify that a document matches the expected format.""" - fileName = document.get("fileName", "") - mimeType = document.get("mimeType", "") - fileSize = document.get("fileSize", 0) - - # Expected MIME types - expectedMimeTypes = { - "html": ["text/html", "application/xhtml+xml"], - "pdf": ["application/pdf"], - "docx": ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"], - "xlsx": ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"], - "pptx": ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] - } - - # Expected file extensions - expectedExtensions = { - "html": [".html", ".htm"], - "pdf": [".pdf"], - "docx": [".docx"], - "xlsx": [".xlsx"], - "pptx": [".pptx"] - } - - formatLower = expectedFormat.lower() - expectedMimes = expectedMimeTypes.get(formatLower, []) - expectedExts = expectedExtensions.get(formatLower, []) - - # Check file extension - hasCorrectExtension = any(fileName.lower().endswith(ext) for ext in expectedExts) - - # Check MIME type - hasCorrectMimeType = any(mimeType.lower() == mime.lower() for mime in expectedMimes) - - # Check file size (should be > 0) - hasValidSize = fileSize > 0 - - verification = { - "format": expectedFormat, - "fileName": fileName, - "mimeType": mimeType, - "fileSize": fileSize, - "hasCorrectExtension": hasCorrectExtension, - "hasCorrectMimeType": hasCorrectMimeType, - "hasValidSize": hasValidSize, - "isValid": hasCorrectExtension and hasValidSize - } - - return verification - - async def testRefactoringFeatures(self) -> Dict[str, Any]: - """Test specific refactoring features.""" - print("\n" + "="*80) - print("TESTING REFACTORING FEATURES") - print("="*80) - - refactoringTests = [ - ("intent_analysis", "html"), - ("conditional_extraction", "html"), - ("image_render", "html"), - ("multi_document", "html"), - ("metadata_preservation", "html") - ] - - results = {} - - for testType, format in refactoringTests: - try: - print(f"\n{'='*80}") - print(f"Testing Refactoring Feature: {testType}") - print(f"{'='*80}") - - prompt = self.createRefactoringTestPrompt(testType, format) - print(f"Prompt: {prompt[:200]}...") - - # Create user input request with PDF file attachment - listFileId = [] - if self.pdfFileId: - listFileId = [self.pdfFileId] - print(f"Attaching PDF file (ID: {self.pdfFileId})") - else: - print("⚠️ No PDF file attached (file upload may have failed)") - - userInput = UserInputRequest( - prompt=prompt, - listFileId=listFileId, - userLanguage="en" - ) - - # Start workflow - print(f"\nStarting workflow for {testType} test...") - workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - if not workflow: - results[testType] = { - "success": False, - "error": "Failed to start workflow" - } - continue - - self.workflow = workflow - print(f"Workflow started: {workflow.id}") - - # Wait for workflow completion (no timeout - wait indefinitely) - completed = await self.waitForWorkflowCompletion(timeout=None) - - if not completed: - results[testType] = { - "success": False, - "error": "Workflow did not complete", - "workflowId": workflow.id - } - continue - - # Analyze results - workflowResults = self.analyzeWorkflowResults() - - # Check for specific refactoring features - verification = self.verifyRefactoringFeature(testType, workflowResults) - - results[testType] = { - "success": True, - "workflowId": workflow.id, - "verification": verification, - "workflowResults": workflowResults - } - - print(f"\n✅ {testType} test completed!") - print(f" Verification: {'✅ PASS' if verification.get('passed', False) else '❌ FAIL'}") - if verification.get("details"): - for detail in verification["details"]: - print(f" - {detail}") - - await asyncio.sleep(2) - - except Exception as e: - import traceback - print(f"\n❌ Error testing {testType}: {str(e)}") - print(traceback.format_exc()) - results[testType] = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - return results - - def verifyRefactoringFeature(self, testType: str, workflowResults: Dict[str, Any]) -> Dict[str, Any]: - """Verify that a refactoring feature works correctly.""" - documents = workflowResults.get("documents", []) - logs = workflowResults.get("logs", []) - - verification = { - "testType": testType, - "passed": False, - "details": [] - } - - if testType == "intent_analysis": - # Check that intent analysis was performed - intentLogs = [log for log in logs if "intent" in str(log).lower() or "analyzing document intent" in str(log).lower()] - if intentLogs: - verification["details"].append("Intent analysis logs found") - verification["passed"] = True - else: - verification["details"].append("No intent analysis logs found") - - elif testType == "conditional_extraction": - # Check that extraction and rendering both occurred - extractionLogs = [log for log in logs if "extract" in str(log).lower()] - renderLogs = [log for log in logs if "render" in str(log).lower() or "image" in str(log).lower()] - if extractionLogs and renderLogs: - verification["details"].append("Both extraction and rendering occurred") - verification["passed"] = True - else: - verification["details"].append(f"Missing logs: extraction={len(extractionLogs)}, render={len(renderLogs)}") - - elif testType == "image_render": - # Check that images were rendered (not extracted as text) - imageLogs = [log for log in logs if "image" in str(log).lower()] - if imageLogs: - verification["details"].append("Image rendering logs found") - verification["passed"] = True - else: - verification["details"].append("No image rendering logs found") - - elif testType == "multi_document": - # Check that multiple documents were generated - if len(documents) >= 2: - verification["details"].append(f"Multiple documents generated: {len(documents)}") - verification["passed"] = True - else: - verification["details"].append(f"Expected multiple documents, got {len(documents)}") - - elif testType == "metadata_preservation": - # Check that metadata was preserved (check logs for documentId references) - metadataLogs = [log for log in logs if "documentId" in str(log) or "SOURCE:" in str(log)] - if metadataLogs: - verification["details"].append("Metadata preservation logs found") - verification["passed"] = True - else: - verification["details"].append("No metadata preservation logs found") - - return verification - - async def testAllFormats(self) -> Dict[str, Any]: - """Test document generation in all formats.""" - print("\n" + "="*80) - print("TESTING DOCUMENT GENERATION IN ALL FORMATS") - print("="*80) - - formats = ["html", "pdf", "docx", "xlsx", "pptx"] - results = {} - - for format in formats: - try: - print(f"\n{'='*80}") - print(f"Testing {format.upper()} format...") - print(f"{'='*80}") - - result = await self.generateDocumentInFormat(format) - results[format] = result - - if result.get("success"): - documents = result.get("documents", []) - if documents: - # Verify first document - verification = self.verifyDocumentFormat(documents[0], format) - result["verification"] = verification - - print(f"\n✅ {format.upper()} generation successful!") - print(f" Documents: {len(documents)}") - print(f" Verification: {'✅ PASS' if verification['isValid'] else '❌ FAIL'}") - if verification.get("fileName"): - print(f" File: {verification['fileName']}") - print(f" Size: {verification['fileSize']} bytes") - print(f" MIME: {verification['mimeType']}") - else: - print(f"\n⚠️ {format.upper()} generation completed but no documents found") - else: - error = result.get("error", "Unknown error") - print(f"\n❌ {format.upper()} generation failed: {error}") - - # Small delay between tests - await asyncio.sleep(2) - - except Exception as e: - import traceback - print(f"\n❌ Error testing {format.upper()}: {str(e)}") - print(traceback.format_exc()) - results[format] = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - return results - - async def runTest(self, includeRefactoringTests: bool = True): - """Run the complete test. - - Args: - includeRefactoringTests: If True, also run refactoring feature tests - """ - print("\n" + "="*80) - print("DOCUMENT GENERATION FORMATS TEST") - print("="*80) - - try: - # Initialize - await self.initialize() - - # Test refactoring features first (if enabled) - refactoringResults = {} - if includeRefactoringTests: - refactoringResults = await self.testRefactoringFeatures() - - # Test all formats - formatResults = await self.testAllFormats() - - # Summary - print("\n" + "="*80) - print("TEST SUMMARY") - print("="*80) - - # Refactoring tests summary - refactoringSuccessCount = 0 - refactoringFailCount = 0 - if includeRefactoringTests and refactoringResults: - print("\nRefactoring Features:") - for testType, result in refactoringResults.items(): - if result.get("success"): - refactoringSuccessCount += 1 - verification = result.get("verification", {}) - passed = verification.get("passed", False) - statusIcon = "✅" if passed else "⚠️" - print(f"{statusIcon} {testType:25s}: {'PASS' if passed else 'FAIL'}") - else: - refactoringFailCount += 1 - error = result.get("error", "Unknown error") - print(f"❌ {testType:25s}: FAIL - {error}") - print(f"Refactoring Tests: {refactoringSuccessCount} passed, {refactoringFailCount} failed out of {len(refactoringResults)} tests") - - # Format tests summary - print("\nFormat Tests:") - successCount = 0 - failCount = 0 - - for format, result in formatResults.items(): - if result.get("success"): - successCount += 1 - status = "✅ PASS" - docCount = result.get("documentCount", 0) - verification = result.get("verification", {}) - isValid = verification.get("isValid", False) - statusIcon = "✅" if isValid else "⚠️" - print(f"{statusIcon} {format.upper():6s}: {status} - {docCount} document(s)") - else: - failCount += 1 - error = result.get("error", "Unknown error") - print(f"❌ {format.upper():6s}: FAIL - {error}") - - print(f"\nFormat Tests: {successCount} passed, {failCount} failed out of {len(formatResults)} formats") - - # Calculate totals - totalSuccess = successCount + refactoringSuccessCount if includeRefactoringTests else successCount - totalFail = failCount + refactoringFailCount if includeRefactoringTests else failCount - - self.testResults = { - "success": failCount == 0 and (not includeRefactoringTests or refactoringFailCount == 0), - "formatTests": { - "successCount": successCount, - "failCount": failCount, - "totalFormats": len(formatResults), - "results": formatResults - }, - "refactoringTests": { - "successCount": refactoringSuccessCount if includeRefactoringTests else 0, - "failCount": refactoringFailCount if includeRefactoringTests else 0, - "totalTests": len(refactoringResults) if includeRefactoringTests else 0, - "results": refactoringResults if includeRefactoringTests else {} - }, - "totalSuccess": totalSuccess, - "totalFail": totalFail - } - - return self.testResults - - except Exception as e: - import traceback - print(f"\n❌ Test failed with error: {type(e).__name__}: {str(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - self.testResults = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - return self.testResults - - -async def main(): - """Run document generation formats test.""" - tester = DocumentGenerationFormatsTester() - results = await tester.runTest() - - # Print final results as JSON for easy parsing - print("\n" + "="*80) - print("FINAL RESULTS (JSON)") - print("="*80) - print(json.dumps(results, indent=2, default=str)) - - -if __name__ == "__main__": - asyncio.run(main()) - diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py deleted file mode 100644 index 043f0690..00000000 --- a/tests/functional/test10_document_generation_formats.py +++ /dev/null @@ -1,558 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Document Generation Formats Test 10 - Tests document generation in DOCX, XLSX, PPTX, and PDF formats -Tests professional document formats with various content types including tables, images, and structured data. -""" - -import asyncio -import json -import sys -import os -import time -import base64 -from typing import Dict, Any, List, Optional - -# Add the gateway to path (go up 2 levels from tests/functional/) -_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _gateway_path not in sys.path: - sys.path.insert(0, _gateway_path) - -# Import the service initialization -from modules.serviceHub import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User -from modules.workflows.automation import chatStart -import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat - - -class DocumentGenerationFormatsTester10: - def __init__(self): - # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelUam import Mandate - rootInterface = getRootInterface() - self.testUser = rootInterface.currentUser - # Get initial mandate ID for testing (User has no mandateId - use initial mandate) - self.testMandateId = rootInterface.getInitialId(Mandate) - - # Initialize services using the existing system - self.services = getServices(self.testUser, None) # Test user, no workflow - self.workflow = None - self.testResults = {} - self.generatedDocuments = {} - self.pdfFileId = None # Store PDF file ID for reuse - - async def initialize(self): - """Initialize the test environment.""" - # Enable debug file logging for tests - from modules.shared.configuration import APP_CONFIG - APP_CONFIG.set("APP_DEBUG_CHAT_WORKFLOW_ENABLED", True) - - # Set logging level to INFO to see workflow progress - import logging - logging.getLogger().setLevel(logging.INFO) - - print(f"Initialized test with user: {self.testUser.id}") - print(f"Test Mandate ID: {self.testMandateId}") - print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") - - # Upload PDF file for testing - await self.uploadPdfFile() - - async def uploadPdfFile(self): - """Upload the PDF file and store its file ID.""" - pdfPath = os.path.join(os.path.dirname(__file__), "..", "..", "..", "local", "temp", "B2025-02c.pdf") - pdfPath = os.path.abspath(pdfPath) - - if not os.path.exists(pdfPath): - print(f"⚠️ Warning: PDF file not found at {pdfPath}") - print(" Test will continue without PDF attachment") - return - - try: - # Read PDF file - with open(pdfPath, "rb") as f: - pdfContent = f.read() - - # Create file using services.interfaceDbComponent - if not hasattr(self.services, 'interfaceDbComponent') or not self.services.interfaceDbComponent: - print("⚠️ Warning: interfaceDbComponent not available in services") - print(" Test will continue without PDF attachment") - return - - interfaceDbComponent = self.services.interfaceDbComponent - - fileItem = interfaceDbComponent.createFile( - name="B2025-02c.pdf", - mimeType="application/pdf", - content=pdfContent - ) - - # Store file data - interfaceDbComponent.createFileData(fileItem.id, pdfContent) - - self.pdfFileId = fileItem.id - print(f"✅ Uploaded PDF file: {fileItem.fileName} (ID: {self.pdfFileId}, Size: {len(pdfContent)} bytes)") - - except Exception as e: - import traceback - print(f"⚠️ Warning: Failed to upload PDF file: {str(e)}") - print(f" Traceback: {traceback.format_exc()}") - print(" Test will continue without PDF attachment") - - def createTestPrompt(self, format: str) -> str: - """Create a test prompt for document generation in the specified format. - - The prompt requests: - - Professional document structure with title, sections, tables, and images - - Extraction of content from attached PDF - - Structured data presentation appropriate for the format - """ - formatPrompts = { - "docx": ( - "Create a professional Word document about 'Fuel Station Receipt Analysis' with:\n" - "1) A main title\n" - "2) An executive summary paragraph\n" - "3) Extract and include the image from the attached PDF document (B2025-02c.pdf)\n" - "4) A detailed analysis section with:\n" - " - Bullet points of key findings\n" - " - A table summarizing transaction details\n" - "5) A conclusion section with recommendations\n\n" - "Format as a professional DOCX document with proper headings and structure." - ), - "xlsx": ( - "Create an Excel spreadsheet analyzing the fuel station receipt from the attached PDF (B2025-02c.pdf).\n" - "Include:\n" - "1) A summary sheet with key metrics\n" - "2) A detailed data sheet with:\n" - " - Transaction details in rows\n" - " - Columns for: Date, Item, Quantity, Price, Total\n" - " - Proper formatting and headers\n" - "3) A calculations sheet with:\n" - " - VAT calculations\n" - " - Net and gross totals\n\n" - "Format as a professional XLSX spreadsheet with formulas and formatting." - ), - "pptx": ( - "Create a PowerPoint presentation about 'Fuel Station Receipt Analysis' with:\n" - "1) Title slide with main title\n" - "2) Overview slide explaining the receipt analysis\n" - "3) Extract and include the image from the attached PDF document (B2025-02c.pdf)\n" - "4) Analysis slides with:\n" - " - Bullet points of key findings\n" - " - Visual representation of data\n" - "5) Conclusion slide with recommendations\n\n" - "Format as a professional PPTX presentation with consistent styling." - ), - "pdf": ( - "Create a professional PDF document about 'Fuel Station Receipt Analysis' with:\n" - "1) A main title\n" - "2) An introduction paragraph explaining the receipt analysis\n" - "3) Extract and include the image from the attached PDF document (B2025-02c.pdf)\n" - "4) A section analyzing the receipt data with:\n" - " - Bullet points of key findings\n" - " - A table summarizing transaction details\n" - "5) A conclusion paragraph with recommendations\n\n" - "Format as a professional PDF document suitable for printing." - ), - "html": ( - "Create a professional HTML document about 'Fuel Station Receipt Analysis' with:\n" - "1) A main title\n" - "2) An introduction paragraph explaining the receipt analysis\n" - "3) Extract and include the image from the attached PDF document (B2025-02c.pdf)\n" - "4) A section analyzing the receipt data with:\n" - " - Bullet points of key findings\n" - " - A table summarizing transaction details\n" - "5) A conclusion paragraph with recommendations\n\n" - "Format as a professional HTML document with proper styling, responsive design, and embedded CSS." - ) - } - - return formatPrompts.get(format.lower(), formatPrompts["docx"]) - - async def generateDocumentInFormat(self, format: str) -> Dict[str, Any]: - """Generate a document in the specified format using workflow.""" - print("\n" + "="*80) - print(f"GENERATING DOCUMENT IN {format.upper()} FORMAT") - print("="*80) - - prompt = self.createTestPrompt(format) - print(f"Prompt: {prompt[:200]}...") - - # Create user input request with PDF file attachment - listFileId = [] - if self.pdfFileId: - listFileId = [self.pdfFileId] - print(f"Attaching PDF file (ID: {self.pdfFileId})") - else: - print("⚠️ No PDF file attached (file upload may have failed)") - - # Create user input request - userInput = UserInputRequest( - prompt=prompt, - listFileId=listFileId, - userLanguage="en" - ) - - # Start workflow - print(f"\nStarting workflow for {format.upper()} generation...") - workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - if not workflow: - return { - "success": False, - "error": "Failed to start workflow" - } - - self.workflow = workflow - print(f"Workflow started: {workflow.id}") - - # Wait for workflow completion (no timeout - wait indefinitely) - print(f"Waiting for workflow completion...") - completed = await self.waitForWorkflowCompletion(timeout=None) - - if not completed: - return { - "success": False, - "error": "Workflow did not complete", - "workflowId": workflow.id, - "status": workflow.status if workflow else "unknown" - } - - # Analyze results - results = self.analyzeWorkflowResults() - - # Extract documents for this format - documents = results.get("documents", []) - formatDocuments = [d for d in documents if d.get("fileName", "").endswith(f".{format.lower()}")] - - return { - "success": True, - "format": format, - "workflowId": workflow.id, - "status": results.get("status"), - "documentCount": len(formatDocuments), - "documents": formatDocuments, - "results": results - } - - async def waitForWorkflowCompletion(self, timeout: Optional[int] = None, checkInterval: int = 2) -> bool: - """Wait for workflow to complete.""" - if not self.workflow: - return False - - startTime = time.time() - lastStatus = None - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - - if timeout is None: - print("Waiting indefinitely (no timeout)") - - while True: - # Check timeout only if specified - if timeout is not None and time.time() - startTime > timeout: - print(f"\n⏱️ Timeout after {timeout} seconds") - return False - - # Get current workflow status - try: - currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) - if not currentWorkflow: - print("\n❌ Workflow not found") - return False - - currentStatus = currentWorkflow.status - elapsed = int(time.time() - startTime) - - # Print status if it changed - if currentStatus != lastStatus: - print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)") - lastStatus = currentStatus - - # Check if workflow is complete - if currentStatus in ["completed", "stopped", "failed"]: - self.workflow = currentWorkflow - statusIcon = "✅" if currentStatus == "completed" else "❌" - print(f"\n{statusIcon} Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)") - return currentStatus == "completed" - - # Wait before next check - await asyncio.sleep(checkInterval) - - except Exception as e: - print(f"\n⚠️ Error checking workflow status: {str(e)}") - await asyncio.sleep(checkInterval) - - def analyzeWorkflowResults(self) -> Dict[str, Any]: - """Analyze workflow results and extract information.""" - if not self.workflow: - return {"error": "No workflow to analyze"} - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - workflow = interfaceDbChat.getWorkflow(self.workflow.id) - - if not workflow: - return {"error": "Workflow not found"} - - # Get unified chat data - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - - # Count messages - messages = chatData.get("messages", []) - userMessages = [m for m in messages if m.get("role") == "user"] - assistantMessages = [m for m in messages if m.get("role") == "assistant"] - - # Count documents - documents = chatData.get("documents", []) - - # Get logs - logs = chatData.get("logs", []) - - results = { - "workflowId": workflow.id, - "status": workflow.status, - "workflowMode": str(workflow.workflowMode) if hasattr(workflow, 'workflowMode') else None, - "currentRound": workflow.currentRound, - "totalTasks": workflow.totalTasks, - "totalActions": workflow.totalActions, - "messageCount": len(messages), - "userMessageCount": len(userMessages), - "assistantMessageCount": len(assistantMessages), - "documentCount": len(documents), - "logCount": len(logs), - "documents": documents, - "logs": logs - } - - print(f"\nWorkflow Results:") - print(f" Status: {results['status']}") - print(f" Tasks: {results['totalTasks']}") - print(f" Actions: {results['totalActions']}") - print(f" Messages: {results['messageCount']}") - print(f" Documents: {results['documentCount']}") - - # Print document details - if documents: - print(f"\nGenerated Documents:") - for doc in documents: - fileName = doc.get("fileName", "unknown") - fileSize = doc.get("fileSize", 0) - mimeType = doc.get("mimeType", "unknown") - documentType = doc.get("documentType", "N/A") - print(f" - {fileName} ({fileSize} bytes, {mimeType}, type: {documentType})") - - return results - - def verifyDocumentFormat(self, document: Dict[str, Any], expectedFormat: str) -> Dict[str, Any]: - """Verify that a document matches the expected format and contains expected metadata.""" - fileName = document.get("fileName", "") - mimeType = document.get("mimeType", "") - fileSize = document.get("fileSize", 0) - documentType = document.get("documentType") - metadata = document.get("metadata") - - # Expected MIME types - expectedMimeTypes = { - "pdf": ["application/pdf"], - "docx": ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"], - "xlsx": ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"], - "pptx": ["application/vnd.openxmlformats-officedocument.presentationml.presentation"], - "html": ["text/html", "application/xhtml+xml"] - } - - # Expected file extensions - expectedExtensions = { - "pdf": [".pdf"], - "docx": [".docx"], - "xlsx": [".xlsx"], - "pptx": [".pptx"], - "html": [".html", ".htm"] - } - - formatLower = expectedFormat.lower() - expectedMimes = expectedMimeTypes.get(formatLower, []) - expectedExts = expectedExtensions.get(formatLower, []) - - # Check file extension - hasCorrectExtension = any(fileName.lower().endswith(ext) for ext in expectedExts) - - # Check MIME type - hasCorrectMimeType = any(mimeType.lower() == mime.lower() for mime in expectedMimes) - - # Check file size (should be > 0) - hasValidSize = fileSize > 0 - - # Check document type (should be present) - hasDocumentType = documentType is not None - - # Check metadata (should be present) - hasMetadata = metadata is not None and isinstance(metadata, dict) - - verification = { - "format": expectedFormat, - "fileName": fileName, - "mimeType": mimeType, - "fileSize": fileSize, - "documentType": documentType, - "hasMetadata": hasMetadata, - "hasCorrectExtension": hasCorrectExtension, - "hasCorrectMimeType": hasCorrectMimeType, - "hasValidSize": hasValidSize, - "hasDocumentType": hasDocumentType, - "isValid": hasCorrectExtension and hasValidSize and hasCorrectMimeType, - "isComplete": hasCorrectExtension and hasValidSize and hasCorrectMimeType and hasDocumentType and hasMetadata - } - - return verification - - async def testAllFormats(self) -> Dict[str, Any]: - """Test document generation in DOCX, XLSX, PPTX, PDF, and HTML formats.""" - print("\n" + "="*80) - print("TESTING DOCUMENT GENERATION IN ALL FORMATS") - print("="*80) - - # Test all document formats - formats = ["docx", "xlsx", "pptx", "pdf", "html"] - results = {} - - for format in formats: - try: - print(f"\n{'='*80}") - print(f"Testing {format.upper()} format...") - print(f"{'='*80}") - - result = await self.generateDocumentInFormat(format) - results[format] = result - - if result.get("success"): - documents = result.get("documents", []) - if documents: - # Verify first document - verification = self.verifyDocumentFormat(documents[0], format) - result["verification"] = verification - - print(f"\n✅ {format.upper()} generation successful!") - print(f" Documents: {len(documents)}") - print(f" Verification: {'✅ PASS' if verification['isValid'] else '❌ FAIL'}") - print(f" Complete (with metadata): {'✅ YES' if verification['isComplete'] else '❌ NO'}") - if verification.get("fileName"): - print(f" File: {verification['fileName']}") - print(f" Size: {verification['fileSize']} bytes") - print(f" MIME: {verification['mimeType']}") - print(f" Document Type: {verification.get('documentType', 'N/A')}") - print(f" Has Metadata: {'✅' if verification.get('hasMetadata') else '❌'}") - else: - print(f"\n⚠️ {format.upper()} generation completed but no documents found") - else: - error = result.get("error", "Unknown error") - print(f"\n❌ {format.upper()} generation failed: {error}") - - # Small delay between tests - await asyncio.sleep(2) - - except Exception as e: - import traceback - print(f"\n❌ Error testing {format.upper()}: {str(e)}") - print(traceback.format_exc()) - results[format] = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - return results - - async def runTest(self): - """Run the complete test.""" - print("\n" + "="*80) - print("DOCUMENT GENERATION FORMATS TEST 10 - ALL FORMATS") - print("="*80) - - try: - # Initialize - await self.initialize() - - # Test all formats - formatResults = await self.testAllFormats() - - # Summary - print("\n" + "="*80) - print("TEST SUMMARY") - print("="*80) - - # Format tests summary - print("\nFormat Tests:") - successCount = 0 - failCount = 0 - completeCount = 0 # Documents with metadata - - for format, result in formatResults.items(): - if result.get("success"): - successCount += 1 - verification = result.get("verification", {}) - isValid = verification.get("isValid", False) - isComplete = verification.get("isComplete", False) - if isComplete: - completeCount += 1 - statusIcon = "✅" if isValid else "⚠️" - completeIcon = "✅" if isComplete else "❌" - docCount = result.get("documentCount", 0) - print(f"{statusIcon} {format.upper():6s}: {'PASS' if isValid else 'FAIL'} - {docCount} document(s) - Metadata: {completeIcon}") - else: - failCount += 1 - error = result.get("error", "Unknown error") - print(f"❌ {format.upper():6s}: FAIL - {error}") - - print(f"\nFormat Tests: {successCount} passed, {failCount} failed out of {len(formatResults)} formats") - print(f"Complete Documents (with metadata): {completeCount} out of {successCount} successful generations") - - self.testResults = { - "success": failCount == 0, - "formatTests": { - "successCount": successCount, - "failCount": failCount, - "completeCount": completeCount, - "totalFormats": len(formatResults), - "results": formatResults - }, - "totalSuccess": successCount, - "totalFail": failCount - } - - return self.testResults - - except Exception as e: - import traceback - print(f"\n❌ Test failed with error: {type(e).__name__}: {str(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - self.testResults = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - return self.testResults - - -async def main(): - """Run document generation formats test 10.""" - tester = DocumentGenerationFormatsTester10() - results = await tester.runTest() - - # Print final results as JSON for easy parsing - print("\n" + "="*80) - print("FINAL RESULTS (JSON)") - print("="*80) - print(json.dumps(results, indent=2, default=str)) - - -if __name__ == "__main__": - asyncio.run(main()) - diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py deleted file mode 100644 index cf410e71..00000000 --- a/tests/functional/test11_code_generation_formats.py +++ /dev/null @@ -1,559 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Code Generation Formats Test 11 - Tests code generation in JSON, CSV, and XML formats -Tests code generation with structured data formats including validation and formatting. -""" - -import asyncio -import json -import sys -import os -import time -import csv -import io -import xml.etree.ElementTree as ET -from typing import Dict, Any, List, Optional - -# Add the gateway to path (go up 2 levels from tests/functional/) -_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _gateway_path not in sys.path: - sys.path.insert(0, _gateway_path) - -# Import the service initialization -from modules.serviceHub import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User -from modules.workflows.automation import chatStart -import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat - - -class CodeGenerationFormatsTester11: - def __init__(self): - # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelUam import Mandate - rootInterface = getRootInterface() - self.testUser = rootInterface.currentUser - # Get initial mandate ID for testing (User has no mandateId - use initial mandate) - self.testMandateId = rootInterface.getInitialId(Mandate) - - # Initialize services using the existing system - self.services = getServices(self.testUser, None) # Test user, no workflow - self.workflow = None - self.testResults = {} - self.generatedDocuments = {} - - async def initialize(self): - """Initialize the test environment.""" - # Enable debug file logging for tests - from modules.shared.configuration import APP_CONFIG - APP_CONFIG.set("APP_DEBUG_CHAT_WORKFLOW_ENABLED", True) - - # Set logging level to INFO to see workflow progress - import logging - logging.getLogger().setLevel(logging.INFO) - - print(f"Initialized test with user: {self.testUser.id}") - print(f"Test Mandate ID: {self.testMandateId}") - print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") - - def createTestPrompt(self, format: str) -> str: - """Create a test prompt for code generation in the specified format. - - The prompt requests 3 files for each format: - - Structured data generation appropriate for the format - - Proper formatting and validation - """ - formatPrompts = { - "json": ( - "Generate 3 JSON code files for a customer management system:\n" - "1) Create a config.json file with:\n" - " - Application name: 'Customer Manager'\n" - " - Version: '1.0.0'\n" - " - Database settings: host, port, name\n" - " - API settings: baseUrl, timeout\n" - "2) Create a customers.json file with an array of customer objects:\n" - " - Each customer should have: id, name, email, phone, address\n" - " - Include at least 3 sample customers\n" - "3) Create a settings.json file with:\n" - " - Theme settings: darkMode, fontSize, language\n" - " - Notification settings: email, sms, push\n" - " - Feature flags: enableAnalytics, enableReports\n\n" - "Format all files as valid JSON with proper indentation." - ), - "csv": ( - "Generate 3 CSV code files for expense tracking:\n" - "1) Create an expenses.csv file with:\n" - " - Header row: Documentname, Datum, Händler, Kreditkartennummer, Gesamtbetrag, Währung, MWST-Satz\n" - " - Data rows with at least 5 expense entries\n" - " - Use consistent date format (DD.MM.YYYY)\n" - " - Use CHF as currency\n" - " - Use 7.7% as VAT rate\n" - "2) Create a categories.csv file with:\n" - " - Header row: CategoryID, CategoryName, Description, ParentCategory\n" - " - Data rows with at least 8 categories\n" - "3) Create a vendors.csv file with:\n" - " - Header row: VendorID, VendorName, ContactPerson, Email, Phone, Address\n" - " - Data rows with at least 6 vendors\n\n" - "Format all files as valid CSV with proper header row and consistent column count." - ), - "xml": ( - "Generate 3 XML code files for a product catalog:\n" - "1) Create a products.xml file with:\n" - " - Root element: \n" - " - Each product as element with:\n" - " - , , , , \n" - " - Include at least 4 products\n" - "2) Create a categories.xml file with:\n" - " - Root element: \n" - " - Each category as element with:\n" - " - , , , \n" - " - Include at least 5 categories\n" - "3) Create a suppliers.xml file with:\n" - " - Root element: \n" - " - Each supplier as element with:\n" - " - , , ,
\n" - " - Include at least 3 suppliers\n\n" - "Format all files as valid XML with proper indentation and structure." - ) - } - - return formatPrompts.get(format.lower(), formatPrompts["json"]) - - async def generateCodeInFormat(self, format: str) -> Dict[str, Any]: - """Generate code in the specified format using workflow.""" - print("\n" + "="*80) - print(f"GENERATING CODE IN {format.upper()} FORMAT") - print("="*80) - - prompt = self.createTestPrompt(format) - print(f"Prompt: {prompt[:200]}...") - - # Create user input request - userInput = UserInputRequest( - prompt=prompt, - listFileId=[], - userLanguage="en" - ) - - # Start workflow - print(f"\nStarting workflow for {format.upper()} code generation...") - workflow = await chatStart( - currentUser=self.testUser, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, - workflowId=None - ) - - if not workflow: - return { - "success": False, - "error": "Failed to start workflow" - } - - self.workflow = workflow - print(f"Workflow started: {workflow.id}") - - # Wait for workflow completion (no timeout - wait indefinitely) - print(f"Waiting for workflow completion...") - completed = await self.waitForWorkflowCompletion(timeout=None) - - if not completed: - return { - "success": False, - "error": "Workflow did not complete", - "workflowId": workflow.id, - "status": workflow.status if workflow else "unknown" - } - - # Analyze results - results = self.analyzeWorkflowResults() - - # Extract documents for this format - documents = results.get("documents", []) - formatDocuments = [d for d in documents if d.get("fileName", "").endswith(f".{format.lower()}")] - - return { - "success": True, - "format": format, - "workflowId": workflow.id, - "status": results.get("status"), - "documentCount": len(formatDocuments), - "documents": formatDocuments, - "results": results - } - - async def waitForWorkflowCompletion(self, timeout: Optional[int] = None, checkInterval: int = 2) -> bool: - """Wait for workflow to complete.""" - if not self.workflow: - return False - - startTime = time.time() - lastStatus = None - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - - if timeout is None: - print("Waiting indefinitely (no timeout)") - - while True: - # Check timeout only if specified - if timeout is not None and time.time() - startTime > timeout: - print(f"\n⏱️ Timeout after {timeout} seconds") - return False - - # Get current workflow status - try: - currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) - if not currentWorkflow: - print("\n❌ Workflow not found") - return False - - currentStatus = currentWorkflow.status - elapsed = int(time.time() - startTime) - - # Print status if it changed - if currentStatus != lastStatus: - print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)") - lastStatus = currentStatus - - # Check if workflow is complete - if currentStatus in ["completed", "stopped", "failed"]: - self.workflow = currentWorkflow - statusIcon = "✅" if currentStatus == "completed" else "❌" - print(f"\n{statusIcon} Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)") - return currentStatus == "completed" - - # Wait before next check - await asyncio.sleep(checkInterval) - - except Exception as e: - print(f"\n⚠️ Error checking workflow status: {str(e)}") - await asyncio.sleep(checkInterval) - - def analyzeWorkflowResults(self) -> Dict[str, Any]: - """Analyze workflow results and extract information.""" - if not self.workflow: - return {"error": "No workflow to analyze"} - - interfaceDbChat = interfaceDbChat.getInterface(self.testUser) - workflow = interfaceDbChat.getWorkflow(self.workflow.id) - - if not workflow: - return {"error": "Workflow not found"} - - # Get unified chat data - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - - # Count messages - messages = chatData.get("messages", []) - userMessages = [m for m in messages if m.get("role") == "user"] - assistantMessages = [m for m in messages if m.get("role") == "assistant"] - - # Count documents - documents = chatData.get("documents", []) - - # Get logs - logs = chatData.get("logs", []) - - results = { - "workflowId": workflow.id, - "status": workflow.status, - "workflowMode": str(workflow.workflowMode) if hasattr(workflow, 'workflowMode') else None, - "currentRound": workflow.currentRound, - "totalTasks": workflow.totalTasks, - "totalActions": workflow.totalActions, - "messageCount": len(messages), - "userMessageCount": len(userMessages), - "assistantMessageCount": len(assistantMessages), - "documentCount": len(documents), - "logCount": len(logs), - "documents": documents, - "logs": logs - } - - print(f"\nWorkflow Results:") - print(f" Status: {results['status']}") - print(f" Tasks: {results['totalTasks']}") - print(f" Actions: {results['totalActions']}") - print(f" Messages: {results['messageCount']}") - print(f" Documents: {results['documentCount']}") - - # Print document details - if documents: - print(f"\nGenerated Documents:") - for doc in documents: - fileName = doc.get("fileName", "unknown") - fileSize = doc.get("fileSize", 0) - mimeType = doc.get("mimeType", "unknown") - print(f" - {fileName} ({fileSize} bytes, {mimeType})") - - return results - - - def verifyCodeFormat(self, document: Dict[str, Any], expectedFormat: str) -> Dict[str, Any]: - """Verify that a code file matches the expected format and is valid.""" - fileName = document.get("fileName", "") - mimeType = document.get("mimeType", "") - fileSize = document.get("fileSize", 0) - - # Expected MIME types - expectedMimeTypes = { - "json": ["application/json"], - "csv": ["text/csv"], - "xml": ["application/xml", "text/xml"] - } - - # Expected file extensions - expectedExtensions = { - "json": [".json"], - "csv": [".csv"], - "xml": [".xml"] - } - - formatLower = expectedFormat.lower() - expectedMimes = expectedMimeTypes.get(formatLower, []) - expectedExts = expectedExtensions.get(formatLower, []) - - # Check file extension - hasCorrectExtension = any(fileName.lower().endswith(ext) for ext in expectedExts) - - # Check MIME type - hasCorrectMimeType = any(mimeType.lower() == mime.lower() for mime in expectedMimes) - - # Check file size (should be > 0) - hasValidSize = fileSize > 0 - - # Try to read and validate content - isValidContent = False - validationError = None - - try: - # Get file content from fileId - fileId = document.get("fileId") - if fileId and hasattr(self.services, 'interfaceDbComponent'): - fileData = self.services.interfaceDbComponent.getFileData(fileId) - if fileData: - content = fileData.decode('utf-8') if isinstance(fileData, bytes) else fileData - - # Validate format-specific syntax - if formatLower == "json": - try: - json.loads(content) - isValidContent = True - except json.JSONDecodeError as e: - validationError = f"Invalid JSON: {str(e)}" - - elif formatLower == "csv": - try: - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - if len(rows) > 0: - # Check header row exists - headerCount = len(rows[0]) - # Check all rows have same column count - allRowsValid = all(len(row) == headerCount for row in rows) - isValidContent = allRowsValid - if not allRowsValid: - validationError = "CSV rows have inconsistent column counts" - else: - validationError = "CSV file is empty" - except Exception as e: - validationError = f"CSV parsing error: {str(e)}" - - elif formatLower == "xml": - try: - ET.fromstring(content) - isValidContent = True - except ET.ParseError as e: - validationError = f"Invalid XML: {str(e)}" - else: - validationError = "Could not read file data" - else: - validationError = "No fileId available" - - except Exception as e: - validationError = f"Error reading/validating file: {str(e)}" - - verification = { - "format": expectedFormat, - "fileName": fileName, - "mimeType": mimeType, - "fileSize": fileSize, - "hasCorrectExtension": hasCorrectExtension, - "hasCorrectMimeType": hasCorrectMimeType, - "hasValidSize": hasValidSize, - "isValidContent": isValidContent, - "validationError": validationError, - "isValid": hasCorrectExtension and hasValidSize and hasCorrectMimeType, - "isComplete": hasCorrectExtension and hasValidSize and hasCorrectMimeType and isValidContent - } - - return verification - - async def testAllFormats(self) -> Dict[str, Any]: - """Test code generation in JSON, CSV, and XML formats.""" - print("\n" + "="*80) - print("TESTING CODE GENERATION IN ALL FORMATS") - print("="*80) - - # Test all code formats - formats = ["json", "csv", "xml"] - results = {} - - for format in formats: - try: - print(f"\n{'='*80}") - print(f"Testing {format.upper()} format...") - print(f"{'='*80}") - - result = await self.generateCodeInFormat(format) - results[format] = result - - if result.get("success"): - documents = result.get("documents", []) - if documents: - # Verify all documents (expecting 3 files per format) - verifications = [] - for doc in documents: - verification = self.verifyCodeFormat(doc, format) - verifications.append(verification) - - result["verifications"] = verifications - - # Count valid documents - validCount = sum(1 for v in verifications if v.get("isValid")) - contentValidCount = sum(1 for v in verifications if v.get("isValidContent")) - - print(f"\n✅ {format.upper()} generation successful!") - print(f" Documents: {len(documents)} (expected: 3)") - print(f" Valid Format: {validCount}/{len(documents)}") - print(f" Valid Content: {contentValidCount}/{len(documents)}") - - # Print details for each file - for i, verification in enumerate(verifications, 1): - statusIcon = "✅" if verification.get("isValid") else "❌" - contentIcon = "✅" if verification.get("isValidContent") else "❌" - print(f" File {i}: {statusIcon} Format, {contentIcon} Content - {verification.get('fileName', 'unknown')}") - if verification.get("validationError"): - print(f" Error: {verification['validationError']}") - else: - print(f"\n⚠️ {format.upper()} generation completed but no documents found") - else: - error = result.get("error", "Unknown error") - print(f"\n❌ {format.upper()} generation failed: {error}") - - # Small delay between tests - await asyncio.sleep(2) - - except Exception as e: - import traceback - print(f"\n❌ Error testing {format.upper()}: {str(e)}") - print(traceback.format_exc()) - results[format] = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - - return results - - async def runTest(self): - """Run the complete test.""" - print("\n" + "="*80) - print("CODE GENERATION FORMATS TEST 11 - JSON, CSV, XML") - print("="*80) - - try: - # Initialize - await self.initialize() - - # Test all formats - formatResults = await self.testAllFormats() - - # Summary - print("\n" + "="*80) - print("TEST SUMMARY") - print("="*80) - - # Format tests summary - print("\nFormat Tests:") - successCount = 0 - failCount = 0 - completeCount = 0 # Files with valid content - - for format, result in formatResults.items(): - if result.get("success"): - successCount += 1 - verifications = result.get("verifications", []) - docCount = result.get("documentCount", 0) - - # Count valid files - validCount = sum(1 for v in verifications if v.get("isValid")) - contentValidCount = sum(1 for v in verifications if v.get("isValidContent")) - completeCount += contentValidCount - - # Overall status (all files valid) - allValid = len(verifications) > 0 and all(v.get("isValid") for v in verifications) - allContentValid = len(verifications) > 0 and all(v.get("isValidContent") for v in verifications) - - statusIcon = "✅" if allValid else "⚠️" - contentIcon = "✅" if allContentValid else "❌" - - print(f"{statusIcon} {format.upper():6s}: {'PASS' if allValid else 'PARTIAL'} - {docCount} file(s) ({validCount} valid format, {contentValidCount} valid content)") - - # Print errors if any - for v in verifications: - if v.get("validationError"): - print(f" {v.get('fileName', 'unknown')}: {v['validationError']}") - else: - failCount += 1 - error = result.get("error", "Unknown error") - print(f"❌ {format.upper():6s}: FAIL - {error}") - - print(f"\nFormat Tests: {successCount} passed, {failCount} failed out of {len(formatResults)} formats") - print(f"Valid Content Files: {completeCount} total files with valid content") - - self.testResults = { - "success": failCount == 0, - "formatTests": { - "successCount": successCount, - "failCount": failCount, - "completeCount": completeCount, - "totalFormats": len(formatResults), - "results": formatResults - }, - "totalSuccess": successCount, - "totalFail": failCount - } - - return self.testResults - - except Exception as e: - import traceback - print(f"\n❌ Test failed with error: {type(e).__name__}: {str(e)}") - print(f"Traceback:\n{traceback.format_exc()}") - self.testResults = { - "success": False, - "error": str(e), - "traceback": traceback.format_exc() - } - return self.testResults - - -async def main(): - """Run code generation formats test 11.""" - tester = CodeGenerationFormatsTester11() - results = await tester.runTest() - - # Print final results as JSON for easy parsing - print("\n" + "="*80) - print("FINAL RESULTS (JSON)") - print("="*80) - print(json.dumps(results, indent=2, default=str)) - - -if __name__ == "__main__": - asyncio.run(main())