From 04ba89a0e89212d78806041fc5818ab65cb754e0 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 22 Jan 2026 00:23:33 +0100 Subject: [PATCH] before feature container refactory --- app.py | 20 +- modules/auth/authentication.py | 2 +- modules/auth/tokenManager.py | 2 +- modules/auth/tokenRefreshService.py | 4 +- modules/datamodels/FIELD_NAMES.md | 314 --- modules/datamodels/datamodelChatbot.py | 1061 +++++++++ modules/features/realEstate/mainRealEstate.py | 2 +- .../features/workflow/subAutomationUtils.py | 2 +- ...rfaceDbAppObjects.py => interfaceDbApp.py} | 0 modules/interfaces/interfaceDbChat.py | 1963 +++++++++++++++++ modules/interfaces/interfaceDbChatbot.py | 10 +- ...entObjects.py => interfaceDbManagement.py} | 2 +- modules/routes/routeAdmin.py | 2 +- ...kflow.py => routeAdminAutomationEvents.py} | 8 +- ...routeFeatures.py => routeAdminFeatures.py} | 2 +- ...eRbacExport.py => routeAdminRbacExport.py} | 2 +- modules/routes/routeAdminRbacRoles.py | 2 +- .../{routeRbac.py => routeAdminRbacRules.py} | 2 +- modules/routes/routeDataAutomation.py | 2 +- modules/routes/routeDataConnections.py | 4 +- modules/routes/routeDataFiles.py | 30 +- modules/routes/routeDataMandates.py | 22 +- modules/routes/routeDataPrompts.py | 12 +- modules/routes/routeDataUsers.py | 20 +- ...outeWorkflows.py => routeDataWorkflows.py} | 6 +- modules/routes/routeFeatureChatDynamic.py | 4 +- modules/routes/routeFeatureChatbot.py | 4 +- modules/routes/routeFeatureRealEstate.py | 2 +- modules/routes/routeFeatureTrustee.py | 2 +- modules/routes/routeGdpr.py | 2 +- modules/routes/routeInvitations.py | 2 +- modules/routes/routeMessaging.py | 30 +- modules/routes/routeSecurityAdmin.py | 2 +- modules/routes/routeSecurityGoogle.py | 2 +- modules/routes/routeSecurityLocal.py | 2 +- modules/routes/routeSecurityMsft.py | 2 +- modules/routes/routeSharepoint.py | 2 +- modules/services/__init__.py | 6 +- .../mainServiceExtraction.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 6 +- .../processing/shared/placeholderFactory.py | 6 +- tests/functional/test03_ai_operations.py | 14 +- tests/functional/test04_ai_behavior.py | 6 +- .../test05_workflow_with_documents.py | 8 +- .../test06_workflow_prompt_variations.py | 8 +- .../test09_document_generation_formats.py | 8 +- .../test10_document_generation_formats.py | 8 +- .../test11_code_generation_formats.py | 8 +- tests/integration/options/test_options_api.py | 2 +- 49 files changed, 3172 insertions(+), 462 deletions(-) delete mode 100644 modules/datamodels/FIELD_NAMES.md create mode 100644 modules/datamodels/datamodelChatbot.py rename modules/interfaces/{interfaceDbAppObjects.py => interfaceDbApp.py} (100%) create mode 100644 modules/interfaces/interfaceDbChat.py rename modules/interfaces/{interfaceDbComponentObjects.py => interfaceDbManagement.py} (99%) rename modules/routes/{routeFeatureWorkflow.py => routeAdminAutomationEvents.py} (94%) rename modules/routes/{routeFeatures.py => routeAdminFeatures.py} (99%) rename modules/routes/{routeRbacExport.py => routeAdminRbacExport.py} (99%) rename modules/routes/{routeRbac.py => routeAdminRbacRules.py} (99%) rename modules/routes/{routeWorkflows.py => routeDataWorkflows.py} (99%) diff --git a/app.py b/app.py index 472de2a1..6944b144 100644 --- a/app.py +++ b/app.py @@ -20,7 +20,7 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager from modules.features import featuresLifecycle as featuresLifecycle -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface class DailyRotatingFileHandler(RotatingFileHandler): """ @@ -421,8 +421,8 @@ app.include_router(promptRouter) from modules.routes.routeDataConnections import router as connectionsRouter app.include_router(connectionsRouter) -from modules.routes.routeWorkflows import router as workflowRouter -app.include_router(workflowRouter) +from modules.routes.routeDataWorkflows import router as dataWorkflowsRouter +app.include_router(dataWorkflowsRouter) from modules.routes.routeFeatureChatDynamic import router as chatPlaygroundRouter app.include_router(chatPlaygroundRouter) @@ -451,11 +451,11 @@ app.include_router(sharepointRouter) from modules.routes.routeDataAutomation import router as automationRouter app.include_router(automationRouter) -from modules.routes.routeFeatureWorkflow import router as adminAutomationEventsRouter +from modules.routes.routeAdminAutomationEvents import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) -from modules.routes.routeRbac import router as rbacRouter -app.include_router(rbacRouter) +from modules.routes.routeAdminRbacRules import router as rbacAdminRulesRouter +app.include_router(rbacAdminRulesRouter) from modules.routes.routeOptions import router as optionsRouter app.include_router(optionsRouter) @@ -470,14 +470,14 @@ from modules.routes.routeFeatureTrustee import router as trusteeRouter app.include_router(trusteeRouter) # Phase 8: New Feature Routes -from modules.routes.routeFeatures import router as featuresRouter -app.include_router(featuresRouter) +from modules.routes.routeAdminFeatures import router as featuresAdminRouter +app.include_router(featuresAdminRouter) from modules.routes.routeInvitations import router as invitationsRouter app.include_router(invitationsRouter) -from modules.routes.routeRbacExport import router as rbacExportRouter -app.include_router(rbacExportRouter) +from modules.routes.routeAdminRbacExport import router as rbacAdminExportRouter +app.include_router(rbacAdminExportRouter) from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) diff --git a/modules/auth/authentication.py b/modules/auth/authentication.py index f6cf0f0d..c6eaafad 100644 --- a/modules/auth/authentication.py +++ b/modules/auth/authentication.py @@ -21,7 +21,7 @@ from slowapi.util import get_remote_address from modules.shared.configuration import APP_CONFIG from modules.security.rootAccess import getRootDbAppConnector, getRootUser -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, AuthAuthority, AccessLevel from modules.datamodels.datamodelSecurity import Token from modules.datamodels.datamodelRbac import AccessRule diff --git a/modules/auth/tokenManager.py b/modules/auth/tokenManager.py index 322b2e4e..0fd76092 100644 --- a/modules/auth/tokenManager.py +++ b/modules/auth/tokenManager.py @@ -259,7 +259,7 @@ class TokenManager: try: if interface is None: from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() interface = getInterface(rootUser) diff --git a/modules/auth/tokenRefreshService.py b/modules/auth/tokenRefreshService.py index 2d780364..a4f8e402 100644 --- a/modules/auth/tokenRefreshService.py +++ b/modules/auth/tokenRefreshService.py @@ -159,7 +159,7 @@ class TokenRefreshService: # Get user interface from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() root_interface = getInterface(rootUser) @@ -228,7 +228,7 @@ class TokenRefreshService: # Get user interface from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() root_interface = getInterface(rootUser) diff --git a/modules/datamodels/FIELD_NAMES.md b/modules/datamodels/FIELD_NAMES.md deleted file mode 100644 index e75899ed..00000000 --- a/modules/datamodels/FIELD_NAMES.md +++ /dev/null @@ -1,314 +0,0 @@ -| Field Name | Type Pattern | Models Using It | -|------------|--------------|-----------------| -| `accumulatedJsonString` | str | JsonAccumulationState | -| `action` | str | ActionDefinition | -| `actionId` | str | ChatDocument, ChatMessage | -| `actionList` | List | TaskItem | -| `actionMethod` | str | ChatMessage | -| `actionName` | str | ChatMessage | -| `actionNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `actionObjective` | str | ActionDefinition, TaskContext | -| `actionProgress` | str | ChatMessage | -| `actionResult` | Any | TaskResult | -| `active` | bool | AutomationDefinition | -| `additionalData` | Dict | AiResponseMetadata | -| `aiPrompt` | str | AiProcessParameters | -| `allSections` | List | JsonAccumulationState | -| `apiUrl` | str | AiModel | -| `authenticationAuthority` | AuthAuthority | User | -| `authority` | AuthAuthority | UserConnection, Token | -| `availableConnections` | list | TaskContext | -| `availableDocuments` | str | TaskContext | -| `base64Encoded` | bool | ContentMetadata, FileData | -| `bytesSent` | int | ChatStat, AiCallResponse | -| `bytesReceived` | int | ChatStat, AiCallResponse | -| `classes` | List | CodeContentPromptArgs | -| `colorMode` | str | ContentMetadata | -| `compressContext` | bool | AiCallOptions | -| `compressPrompt` | bool | AiCallOptions | -| `condition` | str | SelectionRule | -| `confidence` | float | ReviewResult | -| `connectedAt` | float | UserConnection | -| `connectionId` | str | Token | -| `connectionReference` | str | ActionDefinition | -| `connectorType` | str | AiModel | -| `content` | str | Prompt, ContentItem, AiResponse, AiCallResponse, FilePreview | -| `contentAnalysis` | Dict | Observation | -| `contentParts` | List | AiCallRequest, AiProcessParameters, SectionPromptArgs, ChapterStructurePromptArgs, CodeContentPromptArgs, CodeStructurePromptArgs | -| `contentSize` | str | ObservationPreview | -| `contentValidation` | Dict | Observation | -| `contents` | List | ChatContentExtracted | -| `context` | Dict, AccessRuleContext | TaskHandover, UnderstandingResult, AccessRule | -| `contextInfo` | str | CodeContentPromptArgs | -| `costPer1kTokensInput` | float | AiModel | -| `costPer1kTokensOutput` | float | AiModel | -| `country` | str | AiCallPromptWebSearch | -| `create` | AccessLevel | AccessRule, UserPermissions | -| `created` | str | ObservationPreview | -| `createdAt` | float | Token | -| `creationDate` | float | FileItem | -| `criteriaProgress` | dict | TaskContext | -| `currentAction` | int | ChatWorkflow | -| `currentRound` | int | ChatWorkflow | -| `currentTask` | int | ChatWorkflow | -| `data` | str | ContentItem, FileData | -| `dataType` | str | TaskStep | -| `delete` | AccessLevel | AccessRule, UserPermissions | -| `deliverable` | Dict | TaskDefinition | -| `delivered_summary` | str | ContinuationContext | -| `dependencies` | List | TaskItem, TaskStep, CodeContentPromptArgs | -| `description` | TextMultilingual | Role | -| `details` | str | AuthEvent | -| `detectedComplexity` | str | RequestContext | -| `displayName` | str | AiModel | -| `documentData` | Any | ActionDocument, DocumentData | -| `documentId` | str | ObservationPreview | -| `documentList` | DocumentReferenceList | ActionDefinition, ExtractContentParameters | -| `documentName` | str | ActionDocument, DocumentData | -| `documentReferences` | List | UnderstandingResult | -| `documents` | List | ChatMessage, ActionResult, AiResponse | -| `documentsCount` | int | Observation | -| `documentsLabel` | str | ChatMessage, DocumentExchange | -| `durationSec` | float | ContentMetadata | -| `email` | EmailStr | User | -| `enabled` | bool | Mandate, User | -| `encoding` | str | FilePreview | -| `engine` | str | ChatStat | -| `error` | str | ContentMetadata, ActionResult, ActionItem, TaskItem, TaskResult | -| `errorCount` | int | ChatStat, AiCallResponse | -| `estimatedComplexity` | str | TaskStep | -| `eventId` | str | AutomationDefinition | -| `eventType` | str | AuthEvent | -| `execAction` | str | ActionItem | -| `execMethod` | str | ActionItem | -| `execParameters` | Dict | ActionItem | -| `execResultLabel` | str | ActionItem | -| `executedActions` | list | TaskContext | -| `executionLogs` | List | AutomationDefinition | -| `expectedDocumentFormats` | List | ActionItem | -| `expectedFormats` | List | ChatWorkflow, TaskStep | -| `expectedOutputFormat` | str | RequestContext | -| `expectedOutputType` | str | RequestContext | -| `expiresAt` | float | UserConnection, Token | -| `externalEmail` | EmailStr | UserConnection | -| `externalId` | str | UserConnection | -| `externalUsername` | str | UserConnection | -| `extractionMethod` | str | AiResponseMetadata | -| `extractionOptions` | Any | TaskDefinition, ExtractContentParameters | -| `failedActions` | list | TaskContext | -| `failurePatterns` | list | TaskContext | -| `feedback` | str | TaskResult, TaskItem | -| `fileHash` | str | FileItem | -| `fileId` | str | ChatDocument | -| `fileName` | str | FileItem, FilePreview, ChatDocument | -| `fileSize` | int | FileItem, ChatDocument | -| `fileType` | str | CodeContentPromptArgs | -| `filename` | str | AiResponseMetadata, CodeContentPromptArgs | -| `finishedAt` | float | TaskItem | -| `fps` | float | ContentMetadata | -| `fullName` | str | User | -| `functionCall` | Callable | AiModel | -| `functions` | List | CodeContentPromptArgs | -| `generationHint` | str | SectionPromptArgs | -| `handoverType` | str | TaskHandover | -| `hashedPassword` | str | UserInDB | -| `height` | int | ContentMetadata | -| `hierarchyContext` | str | JsonContinuationContexts | -| `hierarchyContextForPrompt` | str | JsonContinuationContexts | -| `id` | str | *Most models* | -| `improvements` | List | TaskHandover, TaskContext, ReviewResult | -| `incomplete_part` | str | ContinuationContext | -| `inputDocuments` | List | TaskHandover | -| `instruction` | str | AiCallPromptWebSearch, AiCallPromptWebCrawl | -| `intention` | Dict | UnderstandingResult | -| `ipAddress` | str | AuthEvent | -| `isAccumulationMode` | bool | JsonAccumulationState | -| `isAggregation` | bool | SectionPromptArgs | -| `isAvailable` | bool | AiModel | -| `isRegeneration` | bool | TaskContext | -| `isSystemRole` | bool | Role | -| `isText` | bool | FilePreview | -| `item` | str | AccessRule | -| `jsonParsingSuccess` | bool | JsonContinuationContexts | -| `kpis` | List | JsonAccumulationState | -| `label` | str | ContentItem, AutomationDefinition | -| `language` | str | Mandate, User, SectionPromptArgs, AiCallPromptWebSearch | -| `last_complete_part` | str | ContinuationContext | -| `last_raw_json` | str | ContinuationContext | -| `lastActivity` | float | ChatWorkflow | -| `lastChecked` | float | UserConnection | -| `lastParsedResult` | Dict | JsonAccumulationState | -| `lastUpdated` | str | AiModel | -| `learnings` | List | ActionDefinition, TaskContext | -| `listFileId` | List | UserInputRequest | -| `logs` | List | ChatWorkflow | -| `mandateId` | str | ChatWorkflow, FileItem, Prompt, User, AutomationDefinition, Token | -| `maxCost` | float | SelectionRule, AiCallOptions | -| `maxDepth` | int | AiCallPromptWebCrawl | -| `maxNumberPages` | int | AiCallPromptWebSearch | -| `maxParts` | int | AiCallOptions | -| `maxProcessingTime` | int | AiCallOptions | -| `maxSteps` | int | ChatWorkflow | -| `maxTokens` | int | AiModel | -| `maxWidth` | int | AiCallPromptWebCrawl | -| `message` | str | ChatLog, ChatMessage | -| `messageHistory` | List | TaskHandover | -| `messageId` | str | ChatDocument | -| `messages` | List | ChatWorkflow, AiModelCall | -| `metadata` | ContentMetadata, AiResponseMetadata, Dict | ContentItem, AiResponse, AiModelResponse, CodeContentPromptArgs | -| `metCriteria` | List | ReviewResult | -| `method` | str | ActionSelection | -| `mime` | str | ObservationPreview | -| `mimeType` | str | ContentMetadata, FileItem, FilePreview, ChatDocument, ActionDocument, DocumentData | -| `minContextLength` | int | AiModel, SelectionRule | -| `minQualityRating` | int | SelectionRule | -| `missingOutputs` | List | ReviewResult | -| `model` | AiModel | AiModelCall | -| `modelId` | str | AiModelResponse | -| `modelName` | str | AiCallResponse | -| `modified` | str | ObservationPreview | -| `name` | str | Mandate, Prompt, ActionSelection, AiModel, SelectionRule, ObservationPreview | -| `nextAction` | str | ReviewResult | -| `nextActionGuidance` | Dict | TaskContext | -| `nextActionObjective` | str | ReviewResult | -| `nextActionParameters` | Dict | ReviewResult | -| `notes` | List | Observation | -| `objective` | str | TaskStep, TaskDefinition | -| `operationId` | str | ChatLog | -| `operationType` | OperationTypeEnum, str | OperationTypeRating, AiCallOptions, AiResponseMetadata | -| `operationTypes` | List | AiModel, SelectionRule | -| `options` | AiCallOptions | AiCallRequest, AiModelCall | -| `originalPrompt` | str | RequestContext | -| `outputDocuments` | List | TaskHandover | -| `outputFormat` | str | ChapterStructurePromptArgs | -| `overlapContext` | str | JsonContinuationContexts | -| `overlap_context` | str | ContinuationContext | -| `overview` | str | TaskPlan | -| `pages` | int | ContentMetadata | -| `parameters` | Dict | ActionParameters, UnderstandingResult, ActionDefinition | -| `parametersContext` | str | ActionDefinition, TaskContext | -| `parentId` | str | ChatLog | -| `parentMessageId` | str | ChatMessage | -| `performance` | Dict | ChatLog | -| `placeholders` | List, Dict | PromptBundle, AutomationDefinition | -| `priceUsd` | float | ChatStat, AiCallResponse | -| `previews` | List | Observation | -| `previousActionResults` | list | TaskContext | -| `previousHandover` | TaskHandover | TaskContext | -| `previousResults` | List | TaskHandover, TaskContext, ReviewContext | -| `previousReviewResult` | dict | TaskContext | -| `priority` | PriorityEnum | AiModel, SelectionRule, AiCallOptions | -| `process` | str | ChatStat | -| `processDocumentsIndividually` | bool | AiCallOptions | -| `processingMode` | ProcessingModeEnum | AiModel, AiCallOptions | -| `processingTime` | float | ChatStat, ActionItem, TaskItem, AiCallResponse, AiModelResponse | -| `progress` | float | ChatLog | -| `prompt` | str | UserInputRequest, PromptBundle, AiCallPromptImage | -| `publishedAt` | float | ChatMessage | -| `qualityRating` | int | AiModel | -| `qualityRequirements` | Dict | TaskStep | -| `qualityScore` | float | ReviewResult | -| `quality` | str | AiCallPromptImage | -| `rating` | int | OperationTypeRating | -| `read` | AccessLevel | AccessRule, UserPermissions | -| `reason` | str | Token, ReviewResult | -| `reference` | str | ObservationPreview | -| `requiredDocuments` | List | TaskDefinition | -| `requiresAnalysis` | bool | RequestContext | -| `requiresContentGeneration` | bool | TaskDefinition | -| `requiresDocumentAnalysis` | bool | TaskDefinition | -| `requiresDocuments` | bool | RequestContext | -| `requiresWebResearch` | bool | RequestContext, TaskDefinition | -| `researchDepth` | str | AiCallPromptWebSearch | -| `resetToken` | str | UserInDB | -| `resetTokenExpires` | float | UserInDB | -| `result` | str | ActionItem | -| `resultFormat` | str | AiCallOptions | -| `resultLabel` | str | ActionResult, Observation | -| `resultLabels` | Dict | TaskItem | -| `resultType` | str | AiProcessParameters | -| `retryCount` | int | ActionItem, TaskItem, TaskContext | -| `retryMax` | int | ActionItem, TaskItem | -| `revokedAt` | float | Token | -| `revokedBy` | str | Token | -| `role` | str | ChatMessage | -| `roleLabel` | str | Role, AccessRule | -| `roleLabels` | List | User | -| `rollbackOnFailure` | bool | TaskItem | -| `roundNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `safetyMargin` | float | AiCallOptions | -| `schedule` | str | AutomationDefinition | -| `schemaVersion` | str | AiResponseMetadata | -| `section` | Dict | SectionPromptArgs | -| `section_count` | int | ContinuationContext | -| `sectionIndex` | int | SectionPromptArgs | -| `sequenceNr` | int | ChatMessage | -| `sessionId` | str | Token | -| `size` | int, str | ContentMetadata, FilePreview, AiCallPromptImage, ObservationPreview | -| `snippet` | str | ObservationPreview | -| `sourceDocuments` | List | AiResponseMetadata | -| `sourceJson` | Dict | ActionDocument, DocumentData | -| `sourceTask` | str | TaskHandover | -| `speedRating` | int | AiModel | -| `stage1Selection` | dict | TaskContext | -| `startedAt` | float | ChatWorkflow, TaskItem | -| `stats` | List | ChatWorkflow | -| `status` | str, TokenStatus, TaskStatus, ConnectionStatus | ChatLog, ChatWorkflow, ChatMessage, UserConnection, AutomationDefinition, ActionItem, TaskItem, TaskResult, ReviewResult, Token | -| `style` | str | AiCallPromptImage | -| `success` | bool | ChatMessage, ActionResult, Observation, TaskResult, AuthEvent, AiModelResponse | -| `successCriteria` | list | TaskStep | -| `successfulActions` | list | TaskContext | -| `summary` | str | ChatMessage | -| `summaryAllowed` | bool | PromptPlaceholder | -| `taskActions` | list | ReviewContext | -| `taskId` | str | TaskResult, TaskHandover | -| `taskNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `taskProgress` | str | ChatMessage | -| `tasks` | List | ChatWorkflow, TaskPlan, UnderstandingResult | -| `taskStep` | TaskStep | TaskContext, ReviewContext | -| `temperature` | float | AiModel, AiCallOptions | -| `template` | str | AutomationDefinition | -| `template_structure` | str | ContinuationContext | -| `timestamp` | float | ChatLog, ActionItem, TaskHandover, AuthEvent | -| `title` | str | AiResponseMetadata | -| `tokenAccess` | str | Token | -| `tokenExpiresAt` | float | UserConnection | -| `tokenRefresh` | str | Token | -| `tokensUsed` | Dict | AiModelResponse | -| `tokenStatus` | str | UserConnection | -| `tokenType` | str | Token | -| `totalActions` | int | ChatWorkflow | -| `totalTasks` | int | ChatWorkflow | -| `type` | str | ChatLog | -| `typeGroup` | str | ObservationPreview | -| `unmetCriteria` | List | ReviewResult | -| `update` | AccessLevel | AccessRule, UserPermissions | -| `url` | str | AiCallPromptWebCrawl | -| `userAgent` | str | AuthEvent | -| `userId` | str | UserConnection, Token, AuthEvent | -| `userInput` | str | TaskItem | -| `userLanguage` | str | UserInputRequest, RequestContext | -| `userMessage` | str | ActionItem, TaskStep, ReviewResult, TaskPlan, ActionDefinition | -| `userPrompt` | str | SectionPromptArgs, ChapterStructurePromptArgs, CodeContentPromptArgs, CodeStructurePromptArgs | -| `username` | str | User | -| `validationMetadata` | Dict | ActionDocument | -| `version` | str | AiModel | -| `view` | bool | AccessRule, UserPermissions | -| `weight` | float | SelectionRule | -| `width` | int | ContentMetadata | -| `workflow` | ChatWorkflow | TaskContext | -| `workflowId` | str | ChatStat, ChatLog, ChatMessage, ChatWorkflow, TaskItem, TaskContext, ReviewContext | -| `workflowMode` | WorkflowModeEnum | ChatWorkflow | -| `workflowSummary` | str | TaskHandover | - - - ---- - -Can you adapt following fields to Multilingual Fields (`TextMultilingual`): - -| Field Name | Models | -|------------|--------| -| `description` | Role | (is already) - - diff --git a/modules/datamodels/datamodelChatbot.py b/modules/datamodels/datamodelChatbot.py new file mode 100644 index 00000000..45c5c4eb --- /dev/null +++ b/modules/datamodels/datamodelChatbot.py @@ -0,0 +1,1061 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chat models: ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument.""" + +from typing import List, Dict, Any, Optional +from enum import Enum +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class ChatStat(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this stat belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this stat belongs to" + ) + workflowId: Optional[str] = Field( + None, description="Foreign key to workflow (for workflow stats)" + ) + processingTime: Optional[float] = Field( + None, description="Processing time in seconds" + ) + bytesSent: Optional[int] = Field(None, description="Number of bytes sent") + bytesReceived: Optional[int] = Field(None, description="Number of bytes received") + errorCount: Optional[int] = Field(None, description="Number of errors encountered") + process: Optional[str] = Field(None, description="The process that delivers the stats data (e.g. 'action.outlook.readMails', 'ai.process.document.name')") + engine: Optional[str] = Field(None, description="The engine used (e.g. 'ai.anthropic.35', 'ai.tavily.basic', 'renderer.docx')") + priceUsd: Optional[float] = Field(None, description="Calculated price in USD for the operation") + + +registerModelLabels( + "ChatStat", + {"en": "Chat Statistics", "fr": "Statistiques de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, + "bytesReceived": {"en": "Bytes Received", "fr": "Octets reçus"}, + "errorCount": {"en": "Error Count", "fr": "Nombre d'erreurs"}, + "process": {"en": "Process", "fr": "Processus"}, + "engine": {"en": "Engine", "fr": "Moteur"}, + "priceUsd": {"en": "Price USD", "fr": "Prix USD"}, + }, +) + + +class ChatLog(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this log belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this log belongs to" + ) + workflowId: str = Field(description="Foreign key to workflow") + message: str = Field(description="Log message") + type: str = Field(description="Log type (info, warning, error, etc.)") + timestamp: float = Field( + default_factory=getUtcTimestamp, + description="When the log entry was created (UTC timestamp in seconds)", + ) + status: Optional[str] = Field(None, description="Status of the log entry") + progress: Optional[float] = Field( + None, description="Progress indicator (0.0 to 1.0)" + ) + performance: Optional[Dict[str, Any]] = Field( + None, description="Performance metrics" + ) + parentId: Optional[str] = Field( + None, description="Parent operation ID (operationId of parent operation) for hierarchical display" + ) + operationId: Optional[str] = Field( + None, description="Operation ID to group related log entries" + ) + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + + +registerModelLabels( + "ChatLog", + {"en": "Chat Log", "fr": "Journal de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "message": {"en": "Message", "fr": "Message"}, + "type": {"en": "Type", "fr": "Type"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "status": {"en": "Status", "fr": "Statut"}, + "progress": {"en": "Progress", "fr": "Progression"}, + "performance": {"en": "Performance", "fr": "Performance"}, + }, +) + + +class ChatDocument(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this document belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to" + ) + messageId: str = Field(description="Foreign key to message") + fileId: str = Field(description="Foreign key to file") + fileName: str = Field(description="Name of the file") + fileSize: int = Field(description="Size of the file") + mimeType: str = Field(description="MIME type of the file") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + actionId: Optional[str] = Field( + None, description="ID of the action that created this document" + ) + + +registerModelLabels( + "ChatDocument", + {"en": "Chat Document", "fr": "Document de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "messageId": {"en": "Message ID", "fr": "ID du message"}, + "fileId": {"en": "File ID", "fr": "ID du fichier"}, + "fileName": {"en": "File Name", "fr": "Nom du fichier"}, + "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + }, +) + + +class ContentMetadata(BaseModel): + size: int = Field(description="Content size in bytes") + pages: Optional[int] = Field( + None, description="Number of pages for multi-page content" + ) + error: Optional[str] = Field(None, description="Processing error if any") + width: Optional[int] = Field(None, description="Width in pixels for images/videos") + height: Optional[int] = Field( + None, description="Height in pixels for images/videos" + ) + colorMode: Optional[str] = Field(None, description="Color mode") + fps: Optional[float] = Field(None, description="Frames per second for videos") + durationSec: Optional[float] = Field( + None, description="Duration in seconds for media" + ) + mimeType: str = Field(description="MIME type of the content") + base64Encoded: bool = Field(description="Whether the data is base64 encoded") + + +registerModelLabels( + "ContentMetadata", + {"en": "Content Metadata", "fr": "Métadonnées du contenu"}, + { + "size": {"en": "Size", "fr": "Taille"}, + "pages": {"en": "Pages", "fr": "Pages"}, + "error": {"en": "Error", "fr": "Erreur"}, + "width": {"en": "Width", "fr": "Largeur"}, + "height": {"en": "Height", "fr": "Hauteur"}, + "colorMode": {"en": "Color Mode", "fr": "Mode de couleur"}, + "fps": {"en": "FPS", "fr": "IPS"}, + "durationSec": {"en": "Duration", "fr": "Durée"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "base64Encoded": {"en": "Base64 Encoded", "fr": "Encodé en Base64"}, + }, +) + + +class ContentItem(BaseModel): + label: str = Field(description="Content label") + data: str = Field(description="Extracted text content") + metadata: ContentMetadata = Field(description="Content metadata") + + +registerModelLabels( + "ContentItem", + {"en": "Content Item", "fr": "Élément de contenu"}, + { + "label": {"en": "Label", "fr": "Étiquette"}, + "data": {"en": "Data", "fr": "Données"}, + "metadata": {"en": "Metadata", "fr": "Métadonnées"}, + }, +) + + +class ChatContentExtracted(BaseModel): + id: str = Field(description="Reference to source ChatDocument") + contents: List[ContentItem] = Field( + default_factory=list, description="List of content items" + ) + + +registerModelLabels( + "ChatContentExtracted", + {"en": "Extracted Content", "fr": "Contenu extrait"}, + { + "id": {"en": "Object ID", "fr": "ID de l'objet"}, + "contents": {"en": "Contents", "fr": "Contenus"}, + }, +) + + +class ChatMessage(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this message belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this message belongs to" + ) + workflowId: str = Field(description="Foreign key to workflow") + parentMessageId: Optional[str] = Field( + None, description="Parent message ID for threading" + ) + documents: List[ChatDocument] = Field( + default_factory=list, description="Associated documents" + ) + documentsLabel: Optional[str] = Field( + None, description="Label for the set of documents" + ) + message: Optional[str] = Field(None, description="Message content") + summary: Optional[str] = Field( + None, description="Short summary of this message for planning/history" + ) + role: str = Field(description="Role of the message sender") + status: str = Field(description="Status of the message (first, step, last)") + sequenceNr: int = Field( + description="Sequence number of the message (set automatically)" + ) + publishedAt: float = Field( + default_factory=getUtcTimestamp, + description="When the message was published (UTC timestamp in seconds)", + ) + success: Optional[bool] = Field( + None, description="Whether the message processing was successful" + ) + actionId: Optional[str] = Field( + None, description="ID of the action that produced this message" + ) + actionMethod: Optional[str] = Field( + None, description="Method of the action that produced this message" + ) + actionName: Optional[str] = Field( + None, description="Name of the action that produced this message" + ) + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + taskProgress: Optional[str] = Field( + None, description="Task progress status: pending, running, success, fail, retry" + ) + actionProgress: Optional[str] = Field( + None, description="Action progress status: pending, running, success, fail" + ) + + +registerModelLabels( + "ChatMessage", + {"en": "Chat Message", "fr": "Message de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "message": {"en": "Message", "fr": "Message"}, + "summary": {"en": "Summary", "fr": "Résumé"}, + "role": {"en": "Role", "fr": "Rôle"}, + "status": {"en": "Status", "fr": "Statut"}, + "sequenceNr": {"en": "Sequence Number", "fr": "Numéro de séquence"}, + "publishedAt": {"en": "Published At", "fr": "Publié le"}, + "success": {"en": "Success", "fr": "Succès"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + "actionMethod": {"en": "Action Method", "fr": "Méthode de l'action"}, + "actionName": {"en": "Action Name", "fr": "Nom de l'action"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "taskProgress": {"en": "Task Progress", "fr": "Progression de la tâche"}, + "actionProgress": {"en": "Action Progress", "fr": "Progression de l'action"}, + }, +) + + +class WorkflowModeEnum(str, Enum): + WORKFLOW_DYNAMIC = "Dynamic" + WORKFLOW_AUTOMATION = "Automation" + WORKFLOW_CHATBOT = "Chatbot" + WORKFLOW_REACT = "React" # Legacy mode - kept for backward compatibility + + +registerModelLabels( + "WorkflowModeEnum", + {"en": "Workflow Mode", "fr": "Mode de workflow"}, + { + "WORKFLOW_DYNAMIC": {"en": "Dynamic", "fr": "Dynamique"}, + "WORKFLOW_AUTOMATION": {"en": "Automation", "fr": "Automatisation"}, + "WORKFLOW_CHATBOT": {"en": "Chatbot", "fr": "Chatbot"}, + "WORKFLOW_REACT": {"en": "React (Legacy)", "fr": "React (Hérité)"}, + }, +) + + +class ChatWorkflow(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: str = Field(description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, + {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, + {"value": "stopped", "label": {"en": "Stopped", "fr": "Arrêté"}}, + {"value": "error", "label": {"en": "Error", "fr": "Erreur"}}, + ]}) + name: Optional[str] = Field(None, description="Name of the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) + currentRound: int = Field(default=0, description="Current round number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + currentTask: int = Field(default=0, description="Current task number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + currentAction: int = Field(default=0, description="Current action number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + totalTasks: int = Field(default=0, description="Total number of tasks in the workflow", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + totalActions: int = Field(default=0, description="Total number of actions in the workflow", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + lastActivity: float = Field(default_factory=getUtcTimestamp, description="Timestamp of last activity (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + startedAt: float = Field(default_factory=getUtcTimestamp, description="When the workflow started (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + logs: List[ChatLog] = Field(default_factory=list, description="Workflow logs", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + messages: List[ChatMessage] = Field(default_factory=list, description="Messages in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + stats: List[ChatStat] = Field(default_factory=list, description="Workflow statistics list", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + tasks: list = Field(default_factory=list, description="List of tasks in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + workflowMode: WorkflowModeEnum = Field(default=WorkflowModeEnum.WORKFLOW_DYNAMIC, description="Workflow mode selector", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + { + "value": WorkflowModeEnum.WORKFLOW_DYNAMIC.value, + "label": {"en": "Dynamic", "fr": "Dynamique"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_AUTOMATION.value, + "label": {"en": "Automation", "fr": "Automatisation"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_CHATBOT.value, + "label": {"en": "Chatbot", "fr": "Chatbot"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_REACT.value, + "label": {"en": "React (Legacy)", "fr": "React (Hérité)"}, + }, + ]}) + maxSteps: int = Field(default=10, description="Maximum number of iterations in dynamic mode", json_schema_extra={"frontend_type": "integer", "frontend_readonly": False, "frontend_required": False}) + expectedFormats: Optional[List[str]] = Field(None, description="List of expected file format extensions from user request (e.g., ['xlsx', 'pdf']). Extracted during intent analysis.", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + + # Helper methods for execution state management + def getRoundIndex(self) -> int: + """Get current round index""" + return self.currentRound + + def getTaskIndex(self) -> int: + """Get current task index""" + return self.currentTask + + def getActionIndex(self) -> int: + """Get current action index""" + return self.currentAction + + def incrementRound(self): + """Increment round when new user input received""" + self.currentRound += 1 + self.currentTask = 0 + self.currentAction = 0 + + def incrementTask(self): + """Increment task when starting new task in current round""" + self.currentTask += 1 + self.currentAction = 0 + + def incrementAction(self): + """Increment action when executing new action in current task""" + self.currentAction += 1 + + +registerModelLabels( + "ChatWorkflow", + {"en": "Chat Workflow", "fr": "Flux de travail de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "status": {"en": "Status", "fr": "Statut"}, + "name": {"en": "Name", "fr": "Nom"}, + "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, + "currentTask": {"en": "Current Task", "fr": "Tâche actuelle"}, + "currentAction": {"en": "Current Action", "fr": "Action actuelle"}, + "totalTasks": {"en": "Total Tasks", "fr": "Total des tâches"}, + "totalActions": {"en": "Total Actions", "fr": "Total des actions"}, + "lastActivity": {"en": "Last Activity", "fr": "Dernière activité"}, + "startedAt": {"en": "Started At", "fr": "Démarré le"}, + "logs": {"en": "Logs", "fr": "Journaux"}, + "messages": {"en": "Messages", "fr": "Messages"}, + "stats": {"en": "Statistics", "fr": "Statistiques"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "workflowMode": {"en": "Workflow Mode", "fr": "Mode de workflow"}, + "maxSteps": {"en": "Max Steps", "fr": "Étapes max"}, + "expectedFormats": {"en": "Expected Formats", "fr": "Formats attendus"}, + }, +) + + +class UserInputRequest(BaseModel): + prompt: str = Field(description="Prompt for the user") + listFileId: List[str] = Field(default_factory=list, description="List of file IDs") + userLanguage: str = Field(default="en", description="User's preferred language") + workflowId: Optional[str] = Field(None, description="Optional ID of the workflow to continue") + + +registerModelLabels( + "UserInputRequest", + {"en": "User Input Request", "fr": "Demande de saisie utilisateur"}, + { + "prompt": {"en": "Prompt", "fr": "Invite"}, + "listFileId": {"en": "File IDs", "fr": "IDs des fichiers"}, + "userLanguage": {"en": "User Language", "fr": "Langue de l'utilisateur"}, + }, +) + + +class ActionDocument(BaseModel): + """Clear document structure for action results""" + + documentName: str = Field(description="Name of the document") + documentData: Any = Field(description="Content/data of the document") + mimeType: str = Field(description="MIME type of the document") + sourceJson: Optional[Dict[str, Any]] = Field( + None, + description="Source JSON structure (preserved when rendering to xlsx/docx/pdf)" + ) + validationMetadata: Optional[Dict[str, Any]] = Field( + None, + description="Action-specific metadata for content validation (e.g., email recipients, attachments, SharePoint paths)" + ) + + +registerModelLabels( + "ActionDocument", + {"en": "Action Document", "fr": "Document d'action"}, + { + "documentName": {"en": "Document Name", "fr": "Nom du document"}, + "documentData": {"en": "Document Data", "fr": "Données du document"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + }, +) + + +class ActionResult(BaseModel): + """Clean action result with documents as primary output + + IMPORTANT: Action methods should NOT set resultLabel in their return value. + The resultLabel is managed by the action handler using the action's execResultLabel + from the action plan. This ensures consistent document routing throughout the workflow. + """ + + success: bool = Field(description="Whether execution succeeded") + error: Optional[str] = Field(None, description="Error message if failed") + documents: List[ActionDocument] = Field( + default_factory=list, description="Document outputs" + ) + resultLabel: Optional[str] = Field( + None, + description="Label for document routing (set by action handler, not by action methods)", + ) + + @classmethod + def isSuccess(cls, documents: List[ActionDocument] = None) -> "ActionResult": + return cls(success=True, documents=documents or []) + + @classmethod + def isFailure( + cls, error: str, documents: List[ActionDocument] = None + ) -> "ActionResult": + return cls(success=False, documents=documents or [], error=error) + + +registerModelLabels( + "ActionResult", + {"en": "Action Result", "fr": "Résultat de l'action"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "error": {"en": "Error", "fr": "Erreur"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + }, +) + + +class ActionSelection(BaseModel): + method: str = Field(description="Method to execute (e.g., web, document, ai)") + name: str = Field( + description="Action name within the method (e.g., search, extract)" + ) + + +registerModelLabels( + "ActionSelection", + {"en": "Action Selection", "fr": "Sélection d'action"}, + { + "method": {"en": "Method", "fr": "Méthode"}, + "name": {"en": "Action Name", "fr": "Nom de l'action"}, + }, +) + + +class ActionParameters(BaseModel): + parameters: Dict[str, Any] = Field( + default_factory=dict, description="Parameters to execute the selected action" + ) + + +registerModelLabels( + "ActionParameters", + {"en": "Action Parameters", "fr": "Paramètres d'action"}, + { + "parameters": {"en": "Parameters", "fr": "Paramètres"}, + }, +) + + +class ObservationPreview(BaseModel): + name: str = Field(description="Document name or URL label") + mime: Optional[str] = Field(default=None, description="MIME type or kind (legacy field)") + snippet: Optional[str] = Field(default=None, description="Short snippet or summary") + # Extended metadata fields + mimeType: Optional[str] = Field(default=None, description="MIME type") + size: Optional[str] = Field(default=None, description="File size") + created: Optional[str] = Field(default=None, description="Creation timestamp") + modified: Optional[str] = Field(default=None, description="Modification timestamp") + typeGroup: Optional[str] = Field(default=None, description="Document type group") + documentId: Optional[str] = Field(default=None, description="Document ID") + reference: Optional[str] = Field(default=None, description="Document reference") + contentSize: Optional[str] = Field(default=None, description="Content size indicator") + + +registerModelLabels( + "ObservationPreview", + {"en": "Observation Preview", "fr": "Aperçu d'observation"}, + { + "name": {"en": "Name", "fr": "Nom"}, + "mime": {"en": "MIME", "fr": "MIME"}, + "snippet": {"en": "Snippet", "fr": "Extrait"}, + }, +) + + +class Observation(BaseModel): + success: bool = Field(description="Action execution success flag") + resultLabel: str = Field(description="Deterministic label for produced documents") + documentsCount: int = Field(description="Number of produced documents") + previews: List[ObservationPreview] = Field( + default_factory=list, description="Compact previews of outputs" + ) + notes: List[str] = Field( + default_factory=list, description="Short notes or key facts" + ) + # Extended fields for enhanced validation + contentValidation: Optional[Dict[str, Any]] = Field( + default=None, description="Content validation results" + ) + contentAnalysis: Optional[Dict[str, Any]] = Field( + default=None, description="Content analysis results" + ) + + +registerModelLabels( + "Observation", + {"en": "Observation", "fr": "Observation"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + "documentsCount": {"en": "Documents Count", "fr": "Nombre de documents"}, + "previews": {"en": "Previews", "fr": "Aperçus"}, + "notes": {"en": "Notes", "fr": "Notes"}, + }, +) + + +class TaskStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +registerModelLabels( + "TaskStatus", + {"en": "Task Status", "fr": "Statut de la tâche"}, + { + "PENDING": {"en": "Pending", "fr": "En attente"}, + "RUNNING": {"en": "Running", "fr": "En cours"}, + "COMPLETED": {"en": "Completed", "fr": "Terminé"}, + "FAILED": {"en": "Failed", "fr": "Échec"}, + "CANCELLED": {"en": "Cancelled", "fr": "Annulé"}, + }, +) + + +class DocumentExchange(BaseModel): + documentsLabel: str = Field(description="Label for the set of documents") + documents: List[str] = Field( + default_factory=list, description="List of document references" + ) + + +registerModelLabels( + "DocumentExchange", + {"en": "Document Exchange", "fr": "Échange de documents"}, + { + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "documents": {"en": "Documents", "fr": "Documents"}, + }, +) + + +class ActionItem(BaseModel): + id: str = Field(..., description="Action ID") + execMethod: str = Field(..., description="Method to execute") + execAction: str = Field(..., description="Action to perform") + execParameters: Dict[str, Any] = Field( + default_factory=dict, description="Action parameters" + ) + execResultLabel: Optional[str] = Field( + None, description="Label for the set of result documents" + ) + expectedDocumentFormats: Optional[List[Dict[str, str]]] = Field( + None, description="Expected document formats (optional)" + ) + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Action status") + error: Optional[str] = Field(None, description="Error message if action failed") + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + processingTime: Optional[float] = Field( + None, description="Processing time in seconds" + ) + timestamp: float = Field( + ..., description="When the action was executed (UTC timestamp in seconds)" + ) + result: Optional[str] = Field(None, description="Result of the action") + + def setSuccess(self, result: str = None) -> None: + """Set the action as successful with optional result""" + self.status = TaskStatus.COMPLETED + self.error = None + if result is not None: + self.result = result + + def setError(self, error_message: str) -> None: + """Set the action as failed with error message""" + self.status = TaskStatus.FAILED + self.error = error_message + + +registerModelLabels( + "ActionItem", + {"en": "Task Action", "fr": "Action de tâche"}, + { + "id": {"en": "Action ID", "fr": "ID de l'action"}, + "execMethod": {"en": "Method", "fr": "Méthode"}, + "execAction": {"en": "Action", "fr": "Action"}, + "execParameters": {"en": "Parameters", "fr": "Paramètres"}, + "execResultLabel": {"en": "Result Label", "fr": "Label du résultat"}, + "expectedDocumentFormats": { + "en": "Expected Document Formats", + "fr": "Formats de documents attendus", + }, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "result": {"en": "Result", "fr": "Résultat"}, + }, +) + + +class TaskResult(BaseModel): + taskId: str = Field(..., description="Task ID") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + success: bool = Field(..., description="Whether the task was successful") + feedback: Optional[str] = Field(None, description="Task feedback message") + error: Optional[str] = Field(None, description="Error message if task failed") + + +registerModelLabels( + "TaskResult", + {"en": "Task Result", "fr": "Résultat de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "status": {"en": "Status", "fr": "Statut"}, + "success": {"en": "Success", "fr": "Succès"}, + "feedback": {"en": "Feedback", "fr": "Retour"}, + "error": {"en": "Error", "fr": "Erreur"}, + }, +) + + +class TaskItem(BaseModel): + id: str = Field(..., description="Task ID") + workflowId: str = Field(..., description="Workflow ID") + userInput: str = Field(..., description="User input that triggered the task") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + error: Optional[str] = Field(None, description="Error message if task failed") + startedAt: Optional[float] = Field( + None, description="When the task started (UTC timestamp in seconds)" + ) + finishedAt: Optional[float] = Field( + None, description="When the task finished (UTC timestamp in seconds)" + ) + actionList: List[ActionItem] = Field( + default_factory=list, description="List of actions to execute" + ) + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + rollbackOnFailure: bool = Field( + default=True, description="Whether to rollback on failure" + ) + dependencies: List[str] = Field( + default_factory=list, description="List of task IDs this task depends on" + ) + feedback: Optional[str] = Field(None, description="Task feedback message") + processingTime: Optional[float] = Field( + None, description="Total processing time in seconds" + ) + resultLabels: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="Map of result labels to their values" + ) + + +registerModelLabels( + "TaskItem", + {"en": "Task", "fr": "Tâche"}, + { + "id": {"en": "Task ID", "fr": "ID de la tâche"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "userInput": {"en": "User Input", "fr": "Entrée utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "startedAt": {"en": "Started At", "fr": "Démarré à"}, + "finishedAt": {"en": "Finished At", "fr": "Terminé à"}, + "actionList": {"en": "Actions", "fr": "Actions"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + }, +) + + +class TaskStep(BaseModel): + id: str + objective: str + dependencies: Optional[list[str]] = Field(default_factory=list) + successCriteria: Optional[list[str]] = Field(default_factory=list) + estimatedComplexity: Optional[str] = None + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + # Format details extracted from intent analysis + dataType: Optional[str] = Field( + None, description="Expected data type (text, numbers, documents, etc.)" + ) + expectedFormats: Optional[List[str]] = Field( + None, description="Expected output file format extensions (e.g., ['docx', 'pdf', 'xlsx']). Use actual file extensions, not conceptual terms." + ) + qualityRequirements: Optional[Dict[str, Any]] = Field( + None, description="Quality requirements and constraints" + ) + + +registerModelLabels( + "TaskStep", + {"en": "Task Step", "fr": "Étape de tâche"}, + { + "id": {"en": "ID", "fr": "ID"}, + "objective": {"en": "Objective", "fr": "Objectif"}, + "dependencies": {"en": "Dependencies", "fr": "Dépendances"}, + "successCriteria": {"en": "Success Criteria", "fr": "Critères de succès"}, + "estimatedComplexity": { + "en": "Estimated Complexity", + "fr": "Complexité estimée", + }, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + "expectedFormats": {"en": "Expected Formats", "fr": "Formats attendus"}, + }, +) + + +class TaskHandover(BaseModel): + taskId: str = Field(description="Target task ID") + sourceTask: Optional[str] = Field(None, description="Source task ID") + inputDocuments: List[DocumentExchange] = Field( + default_factory=list, description="Available input documents" + ) + outputDocuments: List[DocumentExchange] = Field( + default_factory=list, description="Produced output documents" + ) + context: Dict[str, Any] = Field(default_factory=dict, description="Task context") + previousResults: List[str] = Field( + default_factory=list, description="Previous result summaries" + ) + improvements: List[str] = Field( + default_factory=list, description="Improvement suggestions" + ) + workflowSummary: Optional[str] = Field( + None, description="Summarized workflow context" + ) + messageHistory: List[str] = Field( + default_factory=list, description="Key message summaries" + ) + timestamp: float = Field( + ..., description="When the handover was created (UTC timestamp in seconds)" + ) + handoverType: str = Field( + default="task", description="Type of handover: task, phase, or workflow" + ) + + +registerModelLabels( + "TaskHandover", + {"en": "Task Handover", "fr": "Transfert de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "sourceTask": {"en": "Source Task", "fr": "Tâche source"}, + "inputDocuments": {"en": "Input Documents", "fr": "Documents d'entrée"}, + "outputDocuments": {"en": "Output Documents", "fr": "Documents de sortie"}, + "context": {"en": "Context", "fr": "Contexte"}, + "previousResults": {"en": "Previous Results", "fr": "Résultats précédents"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "workflowSummary": {"en": "Workflow Summary", "fr": "Résumé du workflow"}, + "messageHistory": {"en": "Message History", "fr": "Historique des messages"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "handoverType": {"en": "Handover Type", "fr": "Type de transfert"}, + }, +) + + +class TaskContext(BaseModel): + taskStep: TaskStep + workflow: Optional[ChatWorkflow] = None + workflowId: Optional[str] = None + availableDocuments: Optional[str] = "No documents available" + availableConnections: Optional[list[str]] = Field(default_factory=list) + previousResults: Optional[list[str]] = Field(default_factory=list) + previousHandover: Optional[TaskHandover] = None + improvements: Optional[list[str]] = Field(default_factory=list) + retryCount: Optional[int] = 0 + previousActionResults: Optional[list] = Field(default_factory=list) + previousReviewResult: Optional[dict] = None + isRegeneration: Optional[bool] = False + failurePatterns: Optional[list[str]] = Field(default_factory=list) + failedActions: Optional[list] = Field(default_factory=list) + successfulActions: Optional[list] = Field(default_factory=list) + executedActions: Optional[list] = Field(default_factory=list, description="List of executed actions with action name, parameters, and step number") + criteriaProgress: Optional[dict] = None + + # Stage 2 context fields (NEW) + actionObjective: Optional[str] = Field(None, description="Objective for current action") + parametersContext: Optional[str] = Field(None, description="Context for parameter generation") + learnings: Optional[list[str]] = Field(default_factory=list, description="Learnings from previous actions") + stage1Selection: Optional[dict] = Field(None, description="Stage 1 selection data") + nextActionGuidance: Optional[Dict[str, Any]] = Field(None, description="Guidance for the next action from previous refinement") + + def updateFromSelection(self, selection: Any): + """Update context from Stage 1 selection + + Args: + selection: ActionDefinition instance from Stage 1 + """ + from modules.datamodels.datamodelWorkflow import ActionDefinition + + if isinstance(selection, ActionDefinition): + self.actionObjective = selection.actionObjective + self.parametersContext = selection.parametersContext + self.learnings = selection.learnings if selection.learnings else [] + self.stage1Selection = selection.model_dump() + + def getDocumentReferences(self) -> List[str]: + docs = [] + if self.previousHandover: + for doc_exchange in self.previousHandover.inputDocuments: + docs.extend(doc_exchange.documents) + return list(set(docs)) + + def addImprovement(self, improvement: str) -> None: + if improvement not in (self.improvements or []): + if self.improvements is None: + self.improvements = [] + self.improvements.append(improvement) + + +class ReviewContext(BaseModel): + taskStep: TaskStep + taskActions: Optional[list] = Field(default_factory=list) + actionResults: Optional[list] = Field(default_factory=list) + stepResult: Optional[dict] = Field(default_factory=dict) + workflowId: Optional[str] = None + previousResults: Optional[list[str]] = Field(default_factory=list) + + +class ReviewResult(BaseModel): + status: str + reason: Optional[str] = None + improvements: Optional[list[str]] = Field(default_factory=list) + qualityScore: Optional[float] = Field(default=5.0, description="Quality score (0-10)") + missingOutputs: Optional[list[str]] = Field(default_factory=list) + metCriteria: Optional[list[str]] = Field(default_factory=list) + unmetCriteria: Optional[list[str]] = Field(default_factory=list) + confidence: Optional[float] = 0.5 + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + # NEW: Concrete next action guidance (when status is "continue") + nextAction: Optional[str] = Field( + None, description="Specific action to execute next (e.g., 'ai.convert', 'ai.process', 'ai.reformat')" + ) + nextActionParameters: Optional[Dict[str, Any]] = Field( + None, description="Parameters for the next action (e.g., {'fromFormat': 'json', 'toFormat': 'csv'})" + ) + nextActionObjective: Optional[str] = Field( + None, description="What this specific action will achieve" + ) + + +registerModelLabels( + "ReviewResult", + {"en": "Review Result", "fr": "Résultat de l'évaluation"}, + { + "status": {"en": "Status", "fr": "Statut"}, + "reason": {"en": "Reason", "fr": "Raison"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "qualityScore": {"en": "Quality Score", "fr": "Score de qualité"}, + "missingOutputs": {"en": "Missing Outputs", "fr": "Sorties manquantes"}, + "metCriteria": {"en": "Met Criteria", "fr": "Critères respectés"}, + "unmetCriteria": {"en": "Unmet Criteria", "fr": "Critères non respectés"}, + "confidence": {"en": "Confidence", "fr": "Confiance"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + + +class TaskPlan(BaseModel): + overview: str + tasks: list[TaskStep] + userMessage: Optional[str] = Field( + None, description="Overall user-friendly message for the task plan" + ) + + +registerModelLabels( + "TaskPlan", + {"en": "Task Plan", "fr": "Plan de tâches"}, + { + "overview": {"en": "Overview", "fr": "Aperçu"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + +# Forward references resolved automatically since ChatWorkflow is defined above + + +class PromptPlaceholder(BaseModel): + label: str + content: str + summaryAllowed: bool = Field( + default=False, + description="Whether host may summarize content before sending to AI", + ) + + +registerModelLabels( + "PromptPlaceholder", + {"en": "Prompt Placeholder", "fr": "Espace réservé d'invite"}, + { + "label": {"en": "Label", "fr": "Libellé"}, + "content": {"en": "Content", "fr": "Contenu"}, + "summaryAllowed": {"en": "Summary Allowed", "fr": "Résumé autorisé"}, + }, +) + + +class PromptBundle(BaseModel): + prompt: str + placeholders: List[PromptPlaceholder] = Field(default_factory=list) + + +registerModelLabels( + "PromptBundle", + {"en": "Prompt Bundle", "fr": "Lot d'invite"}, + { + "prompt": {"en": "Prompt", "fr": "Invite"}, + "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, + }, +) + + +class AutomationDefinition(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) + schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ + {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, + {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, + {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} + ]}) + template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) + placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) + active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) + eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + + +registerModelLabels( + "AutomationDefinition", + {"en": "Automation Definition", "fr": "Définition d'automatisation"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "label": {"en": "Label", "fr": "Libellé"}, + "schedule": {"en": "Schedule", "fr": "Planification"}, + "template": {"en": "Template", "fr": "Modèle"}, + "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, + "active": {"en": "Active", "fr": "Actif"}, + "eventId": {"en": "Event ID", "fr": "ID de l'événement"}, + "status": {"en": "Status", "fr": "Statut"}, + "executionLogs": {"en": "Execution Logs", "fr": "Journaux d'exécution"}, + }, +) diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 0d386aac..bc729e0d 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -23,7 +23,7 @@ from modules.datamodels.datamodelRealEstate import ( Land, ) from modules.services import getInterface as getServices -from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector logger = logging.getLogger(__name__) diff --git a/modules/features/workflow/subAutomationUtils.py b/modules/features/workflow/subAutomationUtils.py index 906c9caa..97d28719 100644 --- a/modules/features/workflow/subAutomationUtils.py +++ b/modules/features/workflow/subAutomationUtils.py @@ -3,7 +3,7 @@ """ Utility functions for automation feature. -Moved from interfaces/interfaceDbChatbot.py. +Moved from interfaces/interfaceDbChat.py. """ import json diff --git a/modules/interfaces/interfaceDbAppObjects.py b/modules/interfaces/interfaceDbApp.py similarity index 100% rename from modules/interfaces/interfaceDbAppObjects.py rename to modules/interfaces/interfaceDbApp.py diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py new file mode 100644 index 00000000..d171c2ca --- /dev/null +++ b/modules/interfaces/interfaceDbChat.py @@ -0,0 +1,1963 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Interface to LucyDOM database and AI Connectors. +Uses the JSON connector for data access with added language support. +""" + +import logging +import uuid +import math +from typing import Dict, Any, List, Optional, Union + +import asyncio + +from modules.security.rbac import RbacClass +from modules.datamodels.datamodelRbac import AccessRuleContext +from modules.datamodels.datamodelUam import AccessLevel + +from modules.datamodels.datamodelChat import ( + ChatDocument, + ChatStat, + ChatLog, + ChatMessage, + ChatWorkflow, + WorkflowModeEnum, + AutomationDefinition, + UserInputRequest +) +import json +from modules.datamodels.datamodelUam import User + +# DYNAMIC PART: Connectors to the Interface +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC + +# Basic Configurations +from modules.shared.configuration import APP_CONFIG +logger = logging.getLogger(__name__) + +# Singleton factory for Chat instances +_chatInterfaces = {} + + +def storeDebugMessageAndDocuments(message, currentUser) -> None: + """ + Store message and documents (metadata and file bytes) for debugging purposes. + Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ + - message.json, message_text.txt + - document_###_metadata.json + - document_###_ (actual file bytes) + + Args: + message: ChatMessage object to store + currentUser: Current user for component interface access + """ + try: + import os + from datetime import datetime, UTC + from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir + from modules.interfaces.interfaceDbManagement import getInterface + + # Create base debug directory (use base debug dir, not prompts subdirectory) + baseDebugDir = _getBaseDebugDir() + debug_root = os.path.join(baseDebugDir, 'messages') + _ensureDir(debug_root) + + # Generate timestamp + timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] + + # Create message folder name: m_round_task_action_timestamp + # Use actual values from message, not defaults + round_str = str(message.roundNumber) if message.roundNumber is not None else "0" + task_str = str(message.taskNumber) if message.taskNumber is not None else "0" + action_str = str(message.actionNumber) if message.actionNumber is not None else "0" + message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}" + + message_path = os.path.join(debug_root, message_folder) + os.makedirs(message_path, exist_ok=True) + + # Store message data - use dict() instead of model_dump() for compatibility + message_file = os.path.join(message_path, "message.json") + with open(message_file, "w", encoding="utf-8") as f: + # Convert message to dict manually to avoid model_dump() issues + message_dict = { + "id": message.id, + "workflowId": message.workflowId, + "parentMessageId": message.parentMessageId, + "message": message.message, + "role": message.role, + "status": message.status, + "sequenceNr": message.sequenceNr, + "publishedAt": message.publishedAt, + "roundNumber": message.roundNumber, + "taskNumber": message.taskNumber, + "actionNumber": message.actionNumber, + "documentsLabel": message.documentsLabel, + "actionId": message.actionId, + "actionMethod": message.actionMethod, + "actionName": message.actionName, + "success": message.success, + "documents": [] + } + json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str) + + # Store message content as text + if message.message: + message_text_file = os.path.join(message_path, "message_text.txt") + with open(message_text_file, "w", encoding="utf-8") as f: + f.write(str(message.message)) + + # Store documents if provided + if message.documents and len(message.documents) > 0: + # Group documents by documentsLabel + documents_by_label = {} + for doc in message.documents: + label = message.documentsLabel or 'default' + if label not in documents_by_label: + documents_by_label[label] = [] + documents_by_label[label].append(doc) + + # Create subfolder for each document label + for label, docs in documents_by_label.items(): + # Sanitize label for filesystem + safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip() + safe_label = safe_label.replace(' ', '_') + if not safe_label: + safe_label = "default" + + label_folder = os.path.join(message_path, safe_label) + _ensureDir(label_folder) + + # Store each document + for i, doc in enumerate(docs): + # Create document metadata file + doc_meta = { + "id": doc.id, + "messageId": doc.messageId, + "fileId": doc.fileId, + "fileName": doc.fileName, + "fileSize": doc.fileSize, + "mimeType": doc.mimeType, + "roundNumber": doc.roundNumber, + "taskNumber": doc.taskNumber, + "actionNumber": doc.actionNumber, + "actionId": doc.actionId + } + + doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json") + with open(doc_meta_file, "w", encoding="utf-8") as f: + json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str) + + # Also store the actual file bytes next to metadata for debugging + try: + componentInterface = getInterface(currentUser) + file_bytes = componentInterface.getFileData(doc.fileId) + if file_bytes: + # Build a safe filename preserving original name + safe_name = doc.fileName or f"document_{i+1:03d}" + # Avoid path traversal + safe_name = os.path.basename(safe_name) + doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name) + with open(doc_file_path, "wb") as df: + df.write(file_bytes) + else: + pass + except Exception as e: + pass + + except Exception as e: + # Silent fail - don't break main flow + pass + +class ChatObjects: + """ + Interface to Chat database and AI Connectors. + Uses the JSON connector for data access with added language support. + """ + + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """Initializes the Chat Interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) + """ + # Initialize variables + self.currentUser = currentUser # Store User object directly + self.userId = currentUser.id if currentUser else None + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + self.rbac = None # RBAC interface + + # Initialize services + self._initializeServices() + + # Initialize database + self._initializeDatabase() + + # Set user context if provided + if currentUser: + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) + + # ===== Generic Utility Methods ===== + + def _isObjectField(self, fieldType) -> bool: + """Check if a field type represents a complex object (not a simple type).""" + # Simple scalar types + if fieldType in (str, int, float, bool, type(None)): + return False + + # Everything else is an object + return True + + def _separateObjectFields(self, model_class, data: Dict[str, Any]) -> tuple[Dict[str, Any], Dict[str, Any]]: + """Separate simple fields from object fields based on Pydantic model structure.""" + simpleFields = {} + objectFields = {} + + # Get field information from the Pydantic model + modelFields = model_class.model_fields + + for fieldName, value in data.items(): + # Check if this field should be stored as JSONB in the database + if fieldName in modelFields: + fieldInfo = modelFields[fieldName] + # Pydantic v2 only + fieldType = fieldInfo.annotation + + # Always route relational/object fields to object_fields for separate handling + # These fields are stored in separate normalized tables, not as JSONB + if fieldName in ['documents', 'stats', 'logs', 'messages']: + objectFields[fieldName] = value + continue + + # Check if this is a JSONB field (Dict, List, or complex types) + # Purely type-based detection - no hardcoded field names + if (fieldType == dict or + fieldType == list or + (hasattr(fieldType, '__origin__') and fieldType.__origin__ in (dict, list))): + # Store as JSONB - include in simple_fields for database storage + simpleFields[fieldName] = value + elif isinstance(value, (str, int, float, bool, type(None))): + # Simple scalar types + simpleFields[fieldName] = value + else: + # Complex objects that should be filtered out + objectFields[fieldName] = value + else: + # Field not in model - treat as scalar if simple, otherwise filter out + # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector + if fieldName.startswith("_"): + # Metadata fields should be passed through to connector + simpleFields[fieldName] = value + elif isinstance(value, (str, int, float, bool, type(None))): + simpleFields[fieldName] = value + else: + objectFields[fieldName] = value + + return simpleFields, objectFields + + def _initializeServices(self): + pass + + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) + """ + self.currentUser = currentUser # Store User object directly + self.userId = currentUser.id + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId can be None for sysadmins performing cross-mandate operations + if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): + raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + + # Add language settings + self.userLanguage = currentUser.language # Default user language + + # Initialize RBAC interface + if not self.currentUser: + raise ValueError("User context is required for RBAC") + # Get DbApp connection for RBAC AccessRule queries + from modules.security.rootAccess import getRootDbAppConnector + dbApp = getRootDbAppConnector() + self.rbac = RbacClass(self.db, dbApp=dbApp) + + # Update database context + self.db.updateContext(self.userId) + + def __del__(self): + """Cleanup method to close database connection.""" + if hasattr(self, 'db') and self.db is not None: + try: + self.db.close() + except Exception as e: + logger.error(f"Error closing database connection: {e}") + + + def _initializeDatabase(self): + """Initializes the database connection directly.""" + try: + # Get configuration values with defaults + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_chat" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + # Create database connector directly + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId + ) + + # Initialize database system + self.db.initDbSystem() + + logger.info("Database initialized successfully") + except Exception as e: + logger.error(f"Failed to initialize database: {str(e)}") + raise + + def _initRecords(self): + """Initializes standard records in the database if they don't exist.""" + pass + + + def checkRbacPermission( + self, + modelClass: type, + operation: str, + recordId: Optional[str] = None + ) -> bool: + """ + Check RBAC permission for a specific operation on a table. + + Args: + modelClass: Pydantic model class for the table + operation: Operation to check ('create', 'update', 'delete', 'read') + recordId: Optional record ID for specific record check + + Returns: + Boolean indicating permission + """ + if not self.rbac or not self.currentUser: + return False + + tableName = modelClass.__name__ + permissions = self.rbac.getUserPermissions( + self.currentUser, + AccessRuleContext.DATA, + tableName + ) + + if operation == "create": + return permissions.create != AccessLevel.NONE + elif operation == "update": + return permissions.update != AccessLevel.NONE + elif operation == "delete": + return permissions.delete != AccessLevel.NONE + elif operation == "read": + return permissions.read != AccessLevel.NONE + else: + return False + + def _applyFilters(self, records: List[Dict[str, Any]], filters: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Apply filter criteria to records. + + Supports: + - General search: {"search": "text"} - searches across all text fields + - Field-specific filters: + - Simple: {"status": "running"} - equals match + - With operator: {"status": {"operator": "equals", "value": "running"}} + - Operators: equals, contains, gt, gte, lt, lte, in, notIn, startsWith, endsWith + + Args: + records: List of record dictionaries to filter + filters: Filter criteria dictionary + + Returns: + Filtered list of records + """ + if not filters or not records: + return records + + filtered = [] + + for record in records: + matches = True + + # Handle general search across text fields + if "search" in filters: + search_term = str(filters["search"]).lower() + if search_term: + # Search in all string fields + found = False + for key, value in record.items(): + if isinstance(value, str) and search_term in value.lower(): + found = True + break + elif isinstance(value, (int, float)) and search_term in str(value): + found = True + break + if not found: + matches = False + + # Handle field-specific filters + for field_name, filter_value in filters.items(): + if field_name == "search": + continue # Already handled above + + if field_name not in record: + matches = False + break + + record_value = record.get(field_name) + + # Handle simple value (equals operator) + if not isinstance(filter_value, dict): + if record_value != filter_value: + matches = False + break + continue + + # Handle filter with operator + operator = filter_value.get("operator", "equals") + filter_val = filter_value.get("value") + + if operator in ["equals", "eq"]: + if record_value != filter_val: + matches = False + break + + elif operator == "contains": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if filter_str not in record_str: + matches = False + break + + elif operator == "startsWith": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if not record_str.startswith(filter_str): + matches = False + break + + elif operator == "endsWith": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if not record_str.endswith(filter_str): + matches = False + break + + elif operator == "gt": + try: + record_num = float(record_value) if record_value is not None else float('-inf') + filter_num = float(filter_val) if filter_val is not None else float('-inf') + if record_num <= filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "gte": + try: + record_num = float(record_value) if record_value is not None else float('-inf') + filter_num = float(filter_val) if filter_val is not None else float('-inf') + if record_num < filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "lt": + try: + record_num = float(record_value) if record_value is not None else float('inf') + filter_num = float(filter_val) if filter_val is not None else float('inf') + if record_num >= filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "lte": + try: + record_num = float(record_value) if record_value is not None else float('inf') + filter_num = float(filter_val) if filter_val is not None else float('inf') + if record_num > filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "in": + if not isinstance(filter_val, list): + filter_val = [filter_val] + if record_value not in filter_val: + matches = False + break + + elif operator == "notIn": + if not isinstance(filter_val, list): + filter_val = [filter_val] + if record_value in filter_val: + matches = False + break + + else: + # Unknown operator - default to equals + if record_value != filter_val: + matches = False + break + + if matches: + filtered.append(record) + + return filtered + + def _applySorting(self, records: List[Dict[str, Any]], sortFields: List[Any]) -> List[Dict[str, Any]]: + """Apply multi-level sorting to records using stable sort (sorts from least to most significant field).""" + if not sortFields: + return records + + # Start with a copy to avoid modifying original + sortedRecords = list(records) + + # Sort from least significant to most significant field (reverse order) + # Python's sort is stable, so this creates proper multi-level sorting + for sortField in reversed(sortFields): + # Handle both dict and object formats + if isinstance(sortField, dict): + fieldName = sortField.get("field") + direction = sortField.get("direction", "asc") + else: + fieldName = getattr(sortField, "field", None) + direction = getattr(sortField, "direction", "asc") + + if not fieldName: + continue + + isDesc = (direction == "desc") + + def sortKey(record): + value = record.get(fieldName) + # Handle None values - place them at the end for both directions + if value is None: + # Use a special value that sorts last + return (1, "") # (is_none_flag, empty_value) - sorts after (0, ...) + else: + # Return tuple with type indicator for proper comparison + if isinstance(value, (int, float)): + return (0, value) + elif isinstance(value, str): + return (0, value) + elif isinstance(value, bool): + return (0, value) + else: + return (0, str(value)) + + # Sort with reverse parameter for descending + sortedRecords.sort(key=sortKey, reverse=isDesc) + + return sortedRecords + + # Utilities + + def getInitialId(self, model_class: type) -> Optional[str]: + """Returns the initial ID for a table.""" + return self.db.getInitialId(model_class) + + + + # Workflow methods + + def getWorkflows(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + """ + Returns workflows based on user access level. + Supports optional pagination, sorting, and filtering. + + Args: + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[Dict[str, Any]] + If pagination is provided: PaginatedResult with items and metadata + """ + # Use RBAC filtering with featureInstanceId for instance-level isolation + filteredWorkflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId + ) + + # If no pagination requested, return all items (no sorting - frontend handles it) + if pagination is None: + return filteredWorkflows + + # Apply filtering (if filters provided) + if pagination.filters: + filteredWorkflows = self._applyFilters(filteredWorkflows, pagination.filters) + + # Apply sorting (in order of sortFields) - only if provided by frontend + if pagination.sort: + filteredWorkflows = self._applySorting(filteredWorkflows, pagination.sort) + + # Count total items after filters + totalItems = len(filteredWorkflows) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedWorkflows = filteredWorkflows[startIdx:endIdx] + + return PaginatedResult( + items=pagedWorkflows, + totalItems=totalItems, + totalPages=totalPages + ) + + def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: + """Returns a workflow by ID if user has access.""" + # Use RBAC filtering with featureInstanceId for instance-level isolation + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId + ) + + if not workflows: + return None + + workflow = workflows[0] + try: + # Load related data from normalized tables + logs = self.getLogs(workflowId) + messages = self.getMessages(workflowId) + stats = self.getStats(workflowId) + + # Validate workflow data against ChatWorkflow model + return ChatWorkflow( + id=workflow["id"], + status=workflow.get("status", "running"), + name=workflow.get("name"), + currentRound=workflow.get("currentRound", 0) or 0, + currentTask=workflow.get("currentTask", 0) or 0, + currentAction=workflow.get("currentAction", 0) or 0, + totalTasks=workflow.get("totalTasks", 0) or 0, + totalActions=workflow.get("totalActions", 0) or 0, + lastActivity=workflow.get("lastActivity", getUtcTimestamp()), + startedAt=workflow.get("startedAt", getUtcTimestamp()), + logs=logs, + messages=messages, + stats=stats, + mandateId=workflow.get("mandateId", self.mandateId) + ) + except Exception as e: + logger.error(f"Error validating workflow data: {str(e)}") + return None + + def createWorkflow(self, workflowData: Dict[str, Any]) -> ChatWorkflow: + """Creates a new workflow if user has permission.""" + if not self.checkRbacPermission(ChatWorkflow, "create"): + raise PermissionError("No permission to create workflows") + + # Set timestamp if not present + currentTime = getUtcTimestamp() + if "startedAt" not in workflowData: + workflowData["startedAt"] = currentTime + + if "lastActivity" not in workflowData: + workflowData["lastActivity"] = currentTime + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in workflowData or not workflowData["mandateId"]: + workflowData["mandateId"] = self.mandateId + if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: + workflowData["featureInstanceId"] = self.featureInstanceId + + # Use generic field separation based on ChatWorkflow model + simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) + + # Create workflow in database + created = self.db.recordCreate(ChatWorkflow, simpleFields) + + + # Convert to ChatWorkflow model (empty related data for new workflow) + return ChatWorkflow( + id=created["id"], + status=created.get("status", "running"), + name=created.get("name"), + currentRound=created.get("currentRound", 0) or 0, + currentTask=created.get("currentTask", 0) or 0, + currentAction=created.get("currentAction", 0) or 0, + totalTasks=created.get("totalTasks", 0) or 0, + totalActions=created.get("totalActions", 0) or 0, + lastActivity=created.get("lastActivity", currentTime), + startedAt=created.get("startedAt", currentTime), + logs=[], + messages=[], + stats=[], + mandateId=created.get("mandateId", self.mandateId), + workflowMode=created["workflowMode"], + maxSteps=created.get("maxSteps", 1) + ) + + def updateWorkflow(self, workflowId: str, workflowData: Dict[str, Any]) -> ChatWorkflow: + """Updates a workflow if user has access.""" + # Check if the workflow exists and user has access + workflow = self.getWorkflow(workflowId) + if not workflow: + return None + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to update workflow {workflowId}") + + # Use generic field separation based on ChatWorkflow model + simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) + + # Set update time for main workflow + simpleFields["lastActivity"] = getUtcTimestamp() + + # Update main workflow in database + updated = self.db.recordModify(ChatWorkflow, workflowId, simpleFields) + + # Removed cascade writes for logs/messages/stats during workflow update. + # CUD for child entities must be executed via dedicated service methods. + + # Load fresh data from normalized tables + logs = self.getLogs(workflowId) + messages = self.getMessages(workflowId) + stats = self.getStats(workflowId) + + # Convert to ChatWorkflow model + return ChatWorkflow( + id=updated["id"], + status=updated.get("status", workflow.status), + name=updated.get("name", workflow.name), + currentRound=updated.get("currentRound", workflow.currentRound), + currentTask=updated.get("currentTask", workflow.currentTask), + currentAction=updated.get("currentAction", workflow.currentAction), + totalTasks=updated.get("totalTasks", workflow.totalTasks), + totalActions=updated.get("totalActions", workflow.totalActions), + lastActivity=updated.get("lastActivity", workflow.lastActivity), + startedAt=updated.get("startedAt", workflow.startedAt), + logs=logs, + messages=messages, + stats=stats, + mandateId=updated.get("mandateId", workflow.mandateId) + ) + + def deleteWorkflow(self, workflowId: str) -> bool: + """Deletes a workflow and all related data if user has access.""" + try: + # Check if the workflow exists and user has access + workflow = self.getWorkflow(workflowId) + if not workflow: + return False + + if not self.checkRbacPermission(ChatWorkflow, "delete", workflowId): + raise PermissionError(f"No permission to delete workflow {workflowId}") + + # CASCADE DELETE: Delete all related data first + + # 1. Delete all workflow messages and their related data + messages = self.getMessages(workflowId) + for message in messages: + messageId = message.id + if messageId: + # Delete message stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # Delete message documents (but NOT the files!) + existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + for doc in existing_docs: + self.db.recordDelete(ChatDocument, doc["id"]) + + # Delete the message itself + self.db.recordDelete(ChatMessage, messageId) + + # 2. Delete workflow stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # 3. Delete workflow logs + existing_logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + for log in existing_logs: + self.db.recordDelete(ChatLog, log["id"]) + + # 4. Finally delete the workflow itself + success = self.db.recordDelete(ChatWorkflow, workflowId) + + return success + + except Exception as e: + logger.error(f"Error deleting workflow {workflowId}: {str(e)}") + return False + + + # Message methods + + def getMessages(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatMessage], PaginatedResult]: + """ + Returns messages for a workflow if user has access to the workflow. + Supports optional pagination, sorting, and filtering. + + Args: + workflowId: The workflow ID to get messages for + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[ChatMessage] + If pagination is provided: PaginatedResult with items and metadata + """ + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) + + # Get messages for this workflow from normalized table + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + + # Convert raw messages to dict format for sorting/filtering + messageDicts = [] + for msg in messages: + messageDicts.append({ + "id": msg.get("id"), + "workflowId": msg.get("workflowId"), + "parentMessageId": msg.get("parentMessageId"), + "documentsLabel": msg.get("documentsLabel"), + "message": msg.get("message"), + "role": msg.get("role", "assistant"), + "status": msg.get("status", "step"), + "sequenceNr": msg.get("sequenceNr", 0), + "publishedAt": msg.get("publishedAt", msg.get("timestamp", getUtcTimestamp())), + "success": msg.get("success"), + "actionId": msg.get("actionId"), + "actionMethod": msg.get("actionMethod"), + "actionName": msg.get("actionName"), + "roundNumber": msg.get("roundNumber"), + "taskNumber": msg.get("taskNumber"), + "actionNumber": msg.get("actionNumber"), + "taskProgress": msg.get("taskProgress"), + "actionProgress": msg.get("actionProgress") + }) + + # Apply default sorting by publishedAt if no sort specified + if pagination is None or not pagination.sort: + messageDicts.sort(key=lambda x: x.get("publishedAt", getUtcTimestamp())) + + # Apply filtering (if filters provided) + if pagination and pagination.filters: + messageDicts = self._applyFilters(messageDicts, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination and pagination.sort: + messageDicts = self._applySorting(messageDicts, pagination.sort) + + # If no pagination requested, return all items + if pagination is None: + # Convert messages to ChatMessage objects and load documents + chat_messages = [] + for msg in messageDicts: + # Load documents from normalized documents table + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chat_message = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + chat_messages.append(chat_message) + + return chat_messages + + # Count total items after filters + totalItems = len(messageDicts) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedMessageDicts = messageDicts[startIdx:endIdx] + + # Convert messages to ChatMessage objects and load documents + chat_messages = [] + for msg in pagedMessageDicts: + # Load documents from normalized documents table + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chat_message = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + chat_messages.append(chat_message) + + return PaginatedResult( + items=chat_messages, + totalItems=totalItems, + totalPages=totalPages + ) + + def createMessage(self, messageData: Dict[str, Any]) -> ChatMessage: + """Creates a message for a workflow if user has access.""" + try: + # Ensure ID is present + if "id" not in messageData or not messageData["id"]: + messageData["id"] = f"msg_{uuid.uuid4()}" + # Check required fields + requiredFields = ["id", "workflowId"] + for field in requiredFields: + if field not in messageData: + logger.error(f"Required field '{field}' missing in messageData") + raise ValueError(f"Required field '{field}' missing in message data") + + # Check workflow access + workflowId = messageData["workflowId"] + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Validate that ID is not None + if messageData["id"] is None: + messageData["id"] = f"msg_{uuid.uuid4()}" + logger.warning(f"Automatically generated ID for workflow message: {messageData['id']}") + + # Set status if not present + if "status" not in messageData: + messageData["status"] = "step" # Default status for intermediate messages + + # Ensure role and agentName are present + if "role" not in messageData: + messageData["role"] = "assistant" if messageData.get("agentName") else "user" + + if "agentName" not in messageData: + messageData["agentName"] = "" + + # CRITICAL FIX: Automatically set roundNumber, taskNumber, and actionNumber if not provided + # This ensures messages have the correct progress context when workflows are continued + if "roundNumber" not in messageData: + messageData["roundNumber"] = workflow.currentRound + + if "taskNumber" not in messageData: + messageData["taskNumber"] = workflow.currentTask + + if "actionNumber" not in messageData: + messageData["actionNumber"] = workflow.currentAction + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in messageData or not messageData["mandateId"]: + messageData["mandateId"] = self.mandateId + if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: + messageData["featureInstanceId"] = self.featureInstanceId + + # Use generic field separation based on ChatMessage model + simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) + + # Handle documents separately - they will be stored in normalized documents table + documents_to_create = objectFields.get("documents", []) + + # Create message in normalized table using only simple fields + createdMessage = self.db.recordCreate(ChatMessage, simpleFields) + + + # Create documents in normalized documents table + created_documents = [] + logger.debug(f"Creating {len(documents_to_create)} document(s) for message {createdMessage['id']}") + for idx, doc_data in enumerate(documents_to_create): + try: + # Normalize to plain dict before assignment + if isinstance(doc_data, ChatDocument): + doc_dict = doc_data.model_dump() + elif isinstance(doc_data, dict): + doc_dict = dict(doc_data) + else: + # Attempt to coerce to ChatDocument then dump + try: + doc_dict = ChatDocument(**doc_data).model_dump() + except Exception as e: + logger.error(f"Invalid document data type for message creation (document {idx + 1}/{len(documents_to_create)}): {e}") + continue + + # Ensure messageId is set + doc_dict["messageId"] = createdMessage["id"] + logger.debug(f"Creating document {idx + 1}/{len(documents_to_create)}: fileName={doc_dict.get('fileName', 'unknown')}, fileId={doc_dict.get('fileId', 'unknown')}, messageId={doc_dict.get('messageId', 'unknown')}") + + created_doc = self.createDocument(doc_dict) + if created_doc: + created_documents.append(created_doc) + logger.debug(f"Successfully created document {idx + 1}/{len(documents_to_create)}: {created_doc.fileName} (id: {created_doc.id})") + else: + logger.error(f"Failed to create document {idx + 1}/{len(documents_to_create)}: createDocument returned None for fileName={doc_dict.get('fileName', 'unknown')}") + except Exception as e: + logger.error(f"Error processing document {idx + 1}/{len(documents_to_create)}: {e}", exc_info=True) + + logger.info(f"Created {len(created_documents)}/{len(documents_to_create)} document(s) for message {createdMessage['id']}") + + # Convert to ChatMessage model + chat_message = ChatMessage( + id=createdMessage["id"], + workflowId=createdMessage["workflowId"], + parentMessageId=createdMessage.get("parentMessageId"), + agentName=createdMessage.get("agentName"), + documents=created_documents, + documentsLabel=createdMessage.get("documentsLabel"), + message=createdMessage.get("message"), + role=createdMessage.get("role", "assistant"), + status=createdMessage.get("status", "step"), + sequenceNr=len(workflow.messages) + 1, # Use messages list length for sequence number + publishedAt=createdMessage.get("publishedAt", getUtcTimestamp()), + stats=objectFields.get("stats"), # Use stats from objectFields + roundNumber=createdMessage.get("roundNumber"), + taskNumber=createdMessage.get("taskNumber"), + actionNumber=createdMessage.get("actionNumber"), + success=createdMessage.get("success"), + actionId=createdMessage.get("actionId"), + actionMethod=createdMessage.get("actionMethod"), + actionName=createdMessage.get("actionName") + ) + + # Emit message event for streaming (if event manager is available) + try: + from modules.features.chatbot.eventManager import get_event_manager + event_manager = get_event_manager() + message_timestamp = parseTimestamp(chat_message.publishedAt, default=getUtcTimestamp()) + # Emit message event in exact chatData format: {type, createdAt, item} + asyncio.create_task(event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={ + "type": "message", + "createdAt": message_timestamp, + "item": chat_message.dict() + }, + event_category="chat" + )) + except Exception as e: + # Event manager not available or error - continue without emitting + logger.debug(f"Could not emit message event: {e}") + + # Debug: Store message and documents for debugging - only if debug enabled + storeDebugMessageAndDocuments(chat_message, self.currentUser) + + return chat_message + + except Exception as e: + logger.error(f"Error creating workflow message: {str(e)}") + return None + + def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatMessage]: + """Updates a workflow message if user has access to the workflow.""" + try: + + # Ensure messageId is provided + if not messageId: + logger.error("No messageId provided for updateMessage") + raise ValueError("messageId cannot be empty") + + # Check if message exists in database + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"id": messageId}) + if not messages: + logger.warning(f"Message with ID {messageId} does not exist in database") + + # If message doesn't exist but we have workflowId, create it + if "workflowId" in messageData: + workflowId = messageData.get("workflowId") + + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + logger.info(f"Creating new message with ID {messageId} for workflow {workflowId}") + return self.db.recordCreate(ChatMessage, messageData) + else: + logger.error(f"Workflow ID missing for new message {messageId}") + return None + + # Update existing message + existingMessage = messages[0] + + # Check workflow access + workflowId = existingMessage.get("workflowId") + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Use generic field separation based on ChatMessage model + simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) + + # Ensure required fields present + for key in ["role", "agentName"]: + if key not in simpleFields and key not in existingMessage: + simpleFields[key] = "assistant" if key == "role" else "" + + # Ensure ID is in the dataset + if 'id' not in simpleFields: + simpleFields['id'] = messageId + + # Convert createdAt to startedAt if needed + if "createdAt" in simpleFields and "startedAt" not in simpleFields: + simpleFields["startedAt"] = simpleFields["createdAt"] + del simpleFields["createdAt"] + + # Update the message with simple fields only + updatedMessage = self.db.recordModify(ChatMessage, messageId, simpleFields) + + # Handle object field updates (documents, stats) inline + if 'documents' in objectFields: + documents_data = objectFields['documents'] + try: + for doc_data in documents_data: + # Normalize to dict before mutation + if isinstance(doc_data, ChatDocument): + doc_dict = doc_data.model_dump() + elif isinstance(doc_data, dict): + doc_dict = dict(doc_data) + else: + try: + doc_dict = ChatDocument(**doc_data).model_dump() + except Exception: + logger.error("Invalid document data type for message update") + continue + doc_dict["messageId"] = messageId + self.createDocument(doc_dict) + except Exception as e: + logger.error(f"Error updating message documents: {str(e)}") + if not updatedMessage: + logger.warning(f"Failed to update message {messageId}") + return None + + # Convert to ChatMessage model + return ChatMessage(**updatedMessage) + except Exception as e: + logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) + raise ValueError(f"Error updating message {messageId}: {str(e)}") + + def deleteMessage(self, workflowId: str, messageId: str) -> bool: + """Deletes a workflow message and all related data if user has access to the workflow.""" + try: + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return False + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Check if the message exists + messages = self.getMessages(workflowId) + message = next((m for m in messages if m.get("id") == messageId), None) + + if not message: + logger.warning(f"Message {messageId} for workflow {workflowId} not found") + return False + + # CASCADE DELETE: Delete all related data first + + # 1. Delete message stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # 2. Delete message documents (but NOT the files!) + existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + for doc in existing_docs: + self.db.recordDelete(ChatDocument, doc["id"]) + + # 3. Finally delete the message itself + success = self.db.recordDelete(ChatMessage, messageId) + + return success + + except Exception as e: + logger.error(f"Error deleting message {messageId}: {str(e)}") + return False + + def deleteFileFromMessage(self, workflowId: str, messageId: str, fileId: str) -> bool: + """Removes a file reference from a message if user has access.""" + try: + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return False + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + + # Get documents for this message from normalized table + documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + + if not documents: + logger.warning(f"No documents found for message {messageId}") + return False + + # Find and delete the specific document + removed = False + for doc in documents: + docId = doc.get("id") + fileIdValue = doc.get("fileId") + + # Flexible matching approach + shouldRemove = ( + (docId == fileId) or + (fileIdValue == fileId) or + (isinstance(docId, str) and str(fileId) in docId) or + (isinstance(fileIdValue, str) and str(fileId) in fileIdValue) + ) + + if shouldRemove: + # Delete the document from normalized table + success = self.db.recordDelete(ChatDocument, docId) + if success: + removed = True + else: + logger.warning(f"Failed to delete document {docId}") + + if not removed: + logger.warning(f"No matching file {fileId} found in message {messageId}") + return False + + except Exception as e: + logger.error(f"Error removing file {fileId} from message {messageId}: {str(e)}") + return False + + # Document methods + + def getDocuments(self, messageId: str) -> List[ChatDocument]: + """Returns documents for a message from normalized table.""" + try: + documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + return [ChatDocument(**doc) for doc in documents] + except Exception as e: + logger.error(f"Error getting message documents: {str(e)}") + return [] + + def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: + """Creates a document for a message in normalized table.""" + try: + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in documentData or not documentData["mandateId"]: + documentData["mandateId"] = self.mandateId + if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: + documentData["featureInstanceId"] = self.featureInstanceId + + # Validate and normalize document data to dict + document = ChatDocument(**documentData) + logger.debug(f"Creating document in database: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") + created = self.db.recordCreate(ChatDocument, document.model_dump()) + + if created: + created_doc = ChatDocument(**created) + logger.debug(f"Successfully created document in database: {created_doc.fileName} (id: {created_doc.id})") + return created_doc + else: + logger.error(f"Failed to create document in database: recordCreate returned None for fileName={document.fileName}") + return None + except Exception as e: + logger.error(f"Error creating message document: {str(e)}", exc_info=True) + return None + + + # Log methods + + def getLogs(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatLog], PaginatedResult]: + """ + Returns logs for a workflow if user has access to the workflow. + Supports optional pagination, sorting, and filtering. + + Args: + workflowId: The workflow ID to get logs for + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[ChatLog] + If pagination is provided: PaginatedResult with items and metadata + """ + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) + + # Get logs for this workflow from normalized table + logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + + # Convert raw logs to dict format for sorting/filtering + logDicts = [] + for log in logs: + logDicts.append({ + "id": log.get("id"), + "workflowId": log.get("workflowId"), + "message": log.get("message"), + "type": log.get("type"), + "timestamp": log.get("timestamp", getUtcTimestamp()), + "agentName": log.get("agentName"), + "status": log.get("status"), + "progress": log.get("progress"), + "mandateId": log.get("mandateId"), + "userId": log.get("userId") + }) + + # Apply default sorting by timestamp if no sort specified + if pagination is None or not pagination.sort: + logDicts.sort(key=lambda x: parseTimestamp(x.get("timestamp"), default=0)) + + # Apply filtering (if filters provided) + if pagination and pagination.filters: + logDicts = self._applyFilters(logDicts, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination and pagination.sort: + logDicts = self._applySorting(logDicts, pagination.sort) + + # If no pagination requested, return all items + if pagination is None: + return [ChatLog(**log) for log in logDicts] + + # Count total items after filters + totalItems = len(logDicts) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedLogDicts = logDicts[startIdx:endIdx] + + # Convert to model objects + items = [ChatLog(**log) for log in pagedLogDicts] + + return PaginatedResult( + items=items, + totalItems=totalItems, + totalPages=totalPages + ) + + def createLog(self, logData: Dict[str, Any]) -> ChatLog: + """Creates a log entry for a workflow if user has access.""" + # Check workflow access + workflowId = logData.get("workflowId") + if not workflowId: + logger.error("No workflowId provided for createLog") + return None + + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return None + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + logger.warning(f"No permission to modify workflow {workflowId}") + return None + + # Make sure required fields are present + if "timestamp" not in logData: + logData["timestamp"] = getUtcTimestamp() + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in logData or not logData["mandateId"]: + logData["mandateId"] = self.mandateId + if "featureInstanceId" not in logData or not logData["featureInstanceId"]: + logData["featureInstanceId"] = self.featureInstanceId + + # Add status information if not present + if "status" not in logData and "type" in logData: + if logData["type"] == "error": + logData["status"] = "error" + else: + logData["status"] = "running" + + # Add progress information if not present + if "progress" not in logData: + # Default progress values based on log type (0.0 to 1.0 format) + if logData.get("type") == "info": + logData["progress"] = 0.5 # Default middle progress + elif logData.get("type") == "error": + logData["progress"] = 1.0 # Error state - completed (failed) + elif logData.get("type") == "warning": + logData["progress"] = 0.5 # Default middle progress + + # Validate log data against ChatLog model + try: + log_model = ChatLog(**logData) + except Exception as e: + logger.error(f"Invalid log data: {str(e)}") + return None + + # Create log in normalized table + createdLog = self.db.recordCreate(ChatLog, log_model) + + # Emit log event for streaming (only for chatbot workflows) + # Only emit events for chatbot workflows, not for automation or dynamic workflows + if workflow.workflowMode == WorkflowModeEnum.WORKFLOW_CHATBOT: + try: + from modules.features.chatbot.eventManager import get_event_manager + event_manager = get_event_manager() + log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) + # Emit log event in exact chatData format: {type, createdAt, item} + asyncio.create_task(event_manager.emit_event( + workflowId, + "chatdata", + "New log", + "log", + { + "type": "log", + "createdAt": log_timestamp, + "item": ChatLog(**createdLog).model_dump() + } + )) + except Exception as e: + # Event manager not available or error - continue without emitting + logger.debug(f"Could not emit log event: {e}") + + # Return validated ChatLog instance + return ChatLog(**createdLog) + + # Stats methods + + def getStats(self, workflowId: str) -> List[ChatStat]: + """Returns list of statistics for a workflow if user has access.""" + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + return [] + + # Get stats for this workflow from normalized table + stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) + + if not stats: + return [] + + # Return all stats records sorted by creation time + stats.sort(key=lambda x: x.get("created_at", "")) + return [ChatStat(**stat) for stat in stats] + + + def createStat(self, statData: Dict[str, Any]) -> ChatStat: + """Creates a new stats record and returns it.""" + try: + # Ensure workflowId is present in statData + if "workflowId" not in statData: + raise ValueError("workflowId is required in statData") + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in statData or not statData["mandateId"]: + statData["mandateId"] = self.mandateId + if "featureInstanceId" not in statData or not statData["featureInstanceId"]: + statData["featureInstanceId"] = self.featureInstanceId + + # Validate the stat data against ChatStat model + stat = ChatStat(**statData) + + # Create the stat record in the database + created = self.db.recordCreate(ChatStat, stat) + + # Return the created ChatStat + return ChatStat(**created) + except Exception as e: + logger.error(f"Error creating workflow stat: {str(e)}") + raise + + + def getUnifiedChatData(self, workflowId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: + """ + Returns unified chat data (messages, logs, stats) for a workflow in chronological order. + Uses timestamp-based selective data transfer for efficient polling. + """ + # Check workflow access first + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + return {"items": []} + + # Get all data types and filter in Python (PostgreSQL connector doesn't support $gt operators) + items = [] + + # Get messages + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + for msg in messages: + # Apply timestamp filtering in Python + msgTimestamp = parseTimestamp(msg.get("publishedAt"), default=getUtcTimestamp()) + if afterTimestamp is not None and msgTimestamp <= afterTimestamp: + continue + + # Load documents for each message + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chatMessage = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + # Use publishedAt as the timestamp for chronological ordering + items.append({ + "type": "message", + "createdAt": msgTimestamp, + "item": chatMessage + }) + + # Get logs - return all logs with roundNumber if available + logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + for log in logs: + # Apply timestamp filtering in Python + logTimestamp = parseTimestamp(log.get("timestamp"), default=getUtcTimestamp()) + if afterTimestamp is not None and logTimestamp <= afterTimestamp: + continue + + chatLog = ChatLog(**log) + items.append({ + "type": "log", + "createdAt": logTimestamp, + "item": chatLog + }) + + # Get stats list + stats = self.getStats(workflowId) + for stat in stats: + # Apply timestamp filtering in Python + stat_timestamp = stat.createdAt if hasattr(stat, 'createdAt') else getUtcTimestamp() + if afterTimestamp is not None and stat_timestamp <= afterTimestamp: + continue + + items.append({ + "type": "stat", + "createdAt": stat_timestamp, + "item": stat + }) + + # Sort all items by createdAt timestamp for chronological order + items.sort(key=lambda x: parseTimestamp(x.get("createdAt"), default=0)) + + return {"items": items} + + # ===== Automation Methods ===== + + def _computeAutomationStatus(self, automation: Dict[str, Any]) -> str: + """Compute status field based on eventId presence""" + eventId = automation.get("eventId") + return "Running" if eventId else "Idle" + + def _enrichAutomationsWithUserAndMandate(self, automations: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Batch enrich automations with user names and mandate names for display. + Uses AppObjects interface to fetch users and mandates with proper access control. + """ + if not automations: + return automations + + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + + # Collect all unique user IDs and mandate IDs + userIds = set() + mandateIds = set() + + for automation in automations: + createdBy = automation.get("_createdBy") + if createdBy: + userIds.add(createdBy) + + mandateId = automation.get("mandateId") + if mandateId: + mandateIds.add(mandateId) + + # Use AppObjects interface to fetch users (respects access control) + appInterface = getAppInterface(self.currentUser) + usersMap = {} + if userIds: + for user_id in userIds: + user = appInterface.getUser(user_id) + if user: + usersMap[user_id] = user.username or user.email or user_id + + # Use AppObjects interface to fetch mandates (respects access control) + mandatesMap = {} + if mandateIds: + for mandate_id in mandateIds: + mandate = appInterface.getMandate(mandate_id) + if mandate: + mandatesMap[mandate_id] = mandate.name or mandate_id + + # Enrich each automation with the fetched data + for automation in automations: + createdBy = automation.get("_createdBy") + if createdBy: + automation["_createdByUserName"] = usersMap.get(createdBy, createdBy) + else: + automation["_createdByUserName"] = "-" + + mandateId = automation.get("mandateId") + if mandateId: + automation["mandateName"] = mandatesMap.get(mandateId, mandateId) + else: + automation["mandateName"] = "-" + + return automations + + def _enrichAutomationWithUserAndMandate(self, automation: Dict[str, Any]) -> Dict[str, Any]: + """ + Enrich a single automation with user name and mandate name for display. + For multiple automations, use _enrichAutomationsWithUserAndMandate for better performance. + """ + return self._enrichAutomationsWithUserAndMandate([automation])[0] + + def getAllAutomationDefinitions(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + """ + Returns automation definitions based on user access level. + Supports optional pagination, sorting, and filtering. + Computes status field for each automation. + """ + # Use RBAC filtering + filteredAutomations = getRecordsetWithRBAC(self.db, + AutomationDefinition, + self.currentUser + ) + + # Compute status for each automation and normalize executionLogs + for automation in filteredAutomations: + automation["status"] = self._computeAutomationStatus(automation) + # Ensure executionLogs is always a list, not None + if automation.get("executionLogs") is None: + automation["executionLogs"] = [] + + # Batch enrich with user and mandate names + self._enrichAutomationsWithUserAndMandate(filteredAutomations) + + # If no pagination requested, return all items + if pagination is None: + return filteredAutomations + + # Apply filtering (if filters provided) + if pagination.filters: + filteredAutomations = self._applyFilters(filteredAutomations, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination.sort: + filteredAutomations = self._applySorting(filteredAutomations, pagination.sort) + + # Count total items after filters + totalItems = len(filteredAutomations) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedAutomations = filteredAutomations[startIdx:endIdx] + + return PaginatedResult( + items=pagedAutomations, + totalItems=totalItems, + totalPages=totalPages + ) + + def getAutomationDefinition(self, automationId: str) -> Optional[AutomationDefinition]: + """Returns an automation definition by ID if user has access, with computed status.""" + try: + # Use RBAC filtering + filtered = getRecordsetWithRBAC(self.db, + AutomationDefinition, + self.currentUser, + recordFilter={"id": automationId} + ) + + if not filtered: + return None + + automation = filtered[0] + automation["status"] = self._computeAutomationStatus(automation) + # Ensure executionLogs is always a list, not None + if automation.get("executionLogs") is None: + automation["executionLogs"] = [] + # Enrich with user and mandate names + self._enrichAutomationWithUserAndMandate(automation) + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in automation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting automation definition: {str(e)}") + return None + + def createAutomationDefinition(self, automationData: Dict[str, Any]) -> AutomationDefinition: + """Creates a new automation definition, then triggers sync.""" + try: + # Ensure ID is present + if "id" not in automationData or not automationData["id"]: + automationData["id"] = str(uuid.uuid4()) + + # Ensure mandateId and featureInstanceId are set for proper data isolation + if "mandateId" not in automationData: + automationData["mandateId"] = self.mandateId + if "featureInstanceId" not in automationData: + automationData["featureInstanceId"] = self.featureInstanceId + + # Ensure database connector has correct userId context + # The connector should have been initialized with userId, but ensure it's updated + if self.userId and hasattr(self.db, 'updateContext'): + try: + self.db.updateContext(self.userId) + except Exception as e: + logger.warning(f"Could not update database context: {e}") + + # Note: _createdBy will be set automatically by connector's _saveRecord method + # when _createdAt is not present. We don't need to set it manually here. + # Use generic field separation + simpleFields, objectFields = self._separateObjectFields(AutomationDefinition, automationData) + + # Create automation in database + createdAutomation = self.db.recordCreate(AutomationDefinition, simpleFields) + + # Compute status + createdAutomation["status"] = self._computeAutomationStatus(createdAutomation) + # Ensure executionLogs is always a list, not None + if createdAutomation.get("executionLogs") is None: + createdAutomation["executionLogs"] = [] + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in createdAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating automation definition: {str(e)}") + raise + + def updateAutomationDefinition(self, automationId: str, automationData: Dict[str, Any]) -> AutomationDefinition: + """Updates an automation definition, then triggers sync.""" + try: + # Check access + existing = self.getAutomationDefinition(automationId) + if not existing: + raise PermissionError(f"No access to automation {automationId}") + + if not self.checkRbacPermission(AutomationDefinition, "update", automationId): + raise PermissionError(f"No permission to modify automation {automationId}") + + # Use generic field separation + simpleFields, objectFields = self._separateObjectFields(AutomationDefinition, automationData) + + # Update automation in database + updatedAutomation = self.db.recordModify(AutomationDefinition, automationId, simpleFields) + + # Compute status + updatedAutomation["status"] = self._computeAutomationStatus(updatedAutomation) + # Ensure executionLogs is always a list, not None + if updatedAutomation.get("executionLogs") is None: + updatedAutomation["executionLogs"] = [] + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in updatedAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error updating automation definition: {str(e)}") + raise + + def deleteAutomationDefinition(self, automationId: str) -> bool: + """Deletes an automation definition, then triggers sync.""" + try: + # Check access + existing = self.getAutomationDefinition(automationId) + if not existing: + raise PermissionError(f"No access to automation {automationId}") + + if not self.checkRbacPermission(AutomationDefinition, "delete", automationId): + raise PermissionError(f"No permission to delete automation {automationId}") + + # Remove event if exists + if existing.get("eventId"): + from modules.shared.eventManagement import eventManager + try: + eventManager.remove(existing["eventId"]) + except Exception as e: + logger.warning(f"Error removing event {existing['eventId']}: {str(e)}") + + # Delete automation from database + self.db.recordDelete(AutomationDefinition, automationId) + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + return True + except Exception as e: + logger.error(f"Error deleting automation definition: {str(e)}") + raise + + def getAllAutomationDefinitionsWithRBAC(self, user: User) -> List[Dict[str, Any]]: + """ + Get all automation definitions filtered by RBAC for a specific user. + This method encapsulates getRecordsetWithRBAC() to avoid exposing the connector. + + Args: + user: User object for RBAC filtering + + Returns: + List of automation definition dictionaries filtered by RBAC + """ + return getRecordsetWithRBAC( + self.db, + AutomationDefinition, + user + ) + + async def _notifyAutomationChanged(self): + """Notify registered callbacks about automation changes (decoupled from features).""" + try: + from modules.shared.callbackRegistry import callbackRegistry + # Trigger callbacks without knowing which features are listening + await callbackRegistry.trigger('automation.changed', self) + except Exception as e: + logger.error(f"Error notifying automation change: {str(e)}") + + +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': + """ + Returns a ChatObjects instance for the current user. + Handles initialization of database and records. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). + """ + if not currentUser: + raise ValueError("Invalid user context: user is required") + + effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None + + # Create context key including featureInstanceId for proper isolation + contextKey = f"{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" + + # Create new instance if not exists + if contextKey not in _chatInterfaces: + _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + else: + # Update user context if needed + _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + + return _chatInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbChatbot.py b/modules/interfaces/interfaceDbChatbot.py index c9f87a55..44f124b5 100644 --- a/modules/interfaces/interfaceDbChatbot.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -1,8 +1,8 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Interface to LucyDOM database and AI Connectors. -Uses the JSON connector for data access with added language support. +Interface to Chatbot database and AI Connectors. +Uses the PostgreSQL connector for data access with user/mandate filtering. """ import logging @@ -59,7 +59,7 @@ def storeDebugMessageAndDocuments(message, currentUser) -> None: import os from datetime import datetime, UTC from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir - from modules.interfaces.interfaceDbComponentObjects import getInterface + from modules.interfaces.interfaceDbManagement import getInterface # Create base debug directory (use base debug dir, not prompts subdirectory) baseDebugDir = _getBaseDebugDir() @@ -314,7 +314,7 @@ class ChatObjects: try: # Get configuration values with defaults dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") - dbDatabase = "poweron_chat" + dbDatabase = "poweron_chatbot" dbUser = APP_CONFIG.get("DB_USER") dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) @@ -1668,7 +1668,7 @@ class ChatObjects: if not automations: return automations - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface # Collect all unique user IDs and mandate IDs userIds = set() diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbManagement.py similarity index 99% rename from modules/interfaces/interfaceDbComponentObjects.py rename to modules/interfaces/interfaceDbManagement.py index 72d65839..b514cbf3 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -215,7 +215,7 @@ class ComponentObjects: # Get the root interface to access the initial mandate ID from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() rootInterface = getInterface(rootUser) diff --git a/modules/routes/routeAdmin.py b/modules/routes/routeAdmin.py index a60527e1..878dbd66 100644 --- a/modules/routes/routeAdmin.py +++ b/modules/routes/routeAdmin.py @@ -12,7 +12,7 @@ from fastapi import HTTPException, status from modules.shared.configuration import APP_CONFIG from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface # Static folder setup - using absolute path from app root baseDir = FilePath(__file__).parent.parent.parent # Go up to gateway root diff --git a/modules/routes/routeFeatureWorkflow.py b/modules/routes/routeAdminAutomationEvents.py similarity index 94% rename from modules/routes/routeFeatureWorkflow.py rename to modules/routes/routeAdminAutomationEvents.py index 33b9c0fc..c62977aa 100644 --- a/modules/routes/routeFeatureWorkflow.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User @@ -76,8 +76,8 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + from modules.interfaces.interfaceDbApp import getRootInterface from modules.features.workflow import syncAutomationEvents chatInterface = getChatInterface(currentUser) @@ -127,7 +127,7 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatbot.getInterface(currentUser) + chatInterface = interfaceDbChat.getInterface(currentUser) automation = chatInterface.getAutomationDefinition(automation_id) if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeAdminFeatures.py similarity index 99% rename from modules/routes/routeFeatures.py rename to modules/routes/routeAdminFeatures.py index 0659d919..d62a48e1 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelFeatures import Feature, FeatureInstance -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface logger = logging.getLogger(__name__) diff --git a/modules/routes/routeRbacExport.py b/modules/routes/routeAdminRbacExport.py similarity index 99% rename from modules/routes/routeRbacExport.py rename to modules/routes/routeAdminRbacExport.py index e7cc7204..11932f18 100644 --- a/modules/routes/routeRbacExport.py +++ b/modules/routes/routeAdminRbacExport.py @@ -21,7 +21,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelRbac import Role, AccessRule -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index b7069f86..c5c45963 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -17,7 +17,7 @@ from modules.auth import limiter, requireSysAdmin from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelRbac import Role from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeRbac.py b/modules/routes/routeAdminRbacRules.py similarity index 99% rename from modules/routes/routeRbac.py rename to modules/routes/routeAdminRbacRules.py index 7ab9b229..f16a9bc7 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeAdminRbacRules.py @@ -20,7 +20,7 @@ from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestCon from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 3a3e3ab4..db6affab 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -13,7 +13,7 @@ import logging import json # Import interfaces and models -from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface +from modules.interfaces.interfaceDbChat import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index ea660554..f39b6638 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -21,9 +21,9 @@ from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority, from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.interfaces.interfaceDbApp import getInterface from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp -from modules.interfaces.interfaceDbComponentObjects import ComponentObjects +from modules.interfaces.interfaceDbManagement import ComponentObjects # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 23db7170..66345ce5 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -10,7 +10,7 @@ import json from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelFiles import FileItem, FilePreview from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.datamodels.datamodelUam import User @@ -69,7 +69,7 @@ async def get_files( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllFiles(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -112,17 +112,17 @@ async def upload_file( file.fileName = file.filename """Upload a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Read file fileContent = await file.read() # Check size limits - maxSize = int(interfaceDbComponentObjects.APP_CONFIG.get("File_Management_MAX_UPLOAD_SIZE_MB")) * 1024 * 1024 # in bytes + maxSize = int(interfaceDbManagement.APP_CONFIG.get("File_Management_MAX_UPLOAD_SIZE_MB")) * 1024 * 1024 # in bytes if len(fileContent) > maxSize: raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File too large. Maximum size: {interfaceDbComponentObjects.APP_CONFIG.get('File_Management_MAX_UPLOAD_SIZE_MB')}MB" + detail=f"File too large. Maximum size: {interfaceDbManagement.APP_CONFIG.get('File_Management_MAX_UPLOAD_SIZE_MB')}MB" ) # Save file via LucyDOM interface in the database @@ -155,7 +155,7 @@ async def upload_file( "isDuplicate": duplicateType != "new_file" }) - except interfaceDbComponentObjects.FileStorageError as e: + except interfaceDbManagement.FileStorageError as e: logger.error(f"Error during file upload (storage): {str(e)}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -177,7 +177,7 @@ async def get_file( ) -> FileItem: """Get a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file via LucyDOM interface from the database fileData = managementInterface.getFile(fileId) @@ -189,19 +189,19 @@ async def get_file( return fileData - except interfaceDbComponentObjects.FileNotFoundError as e: + except interfaceDbManagement.FileNotFoundError as e: logger.warning(f"File not found: {str(e)}") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=str(e) ) - except interfaceDbComponentObjects.FilePermissionError as e: + except interfaceDbManagement.FilePermissionError as e: logger.warning(f"No permission for file: {str(e)}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=str(e) ) - except interfaceDbComponentObjects.FileError as e: + except interfaceDbManagement.FileError as e: logger.error(f"Error retrieving file: {str(e)}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -224,7 +224,7 @@ async def update_file( ) -> FileItem: """Update file info""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get the file from the database file = managementInterface.getFile(fileId) @@ -270,7 +270,7 @@ async def delete_file( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a file""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the file exists existingFile = managementInterface.getFile(fileId) @@ -297,7 +297,7 @@ async def get_file_stats( ) -> Dict[str, Any]: """Returns statistics about the stored files""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get all files - metadata only allFiles = managementInterface.getAllFiles() @@ -336,7 +336,7 @@ async def download_file( ) -> Response: """Download a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file data fileData = managementInterface.getFile(fileId) @@ -384,7 +384,7 @@ async def preview_file( ) -> FilePreview: """Preview a file's content""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file preview using the correct method preview = managementInterface.getFileContent(fileId) diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 82ed3ad6..ab0ad8c1 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -20,7 +20,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, requireSysAdmin, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects +import modules.interfaces.interfaceDbApp as interfaceDbApp from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.shared.auditLogger import audit_logger @@ -107,7 +107,7 @@ async def get_mandates( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() result = appInterface.getAllMandates(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -150,7 +150,7 @@ async def get_mandate( MULTI-TENANT: SysAdmin-only. """ try: - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() mandate = appInterface.getMandate(mandateId) if not mandate: @@ -195,7 +195,7 @@ async def create_mandate( description = mandateData.get('description') enabled = mandateData.get('enabled', True) - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Create mandate newMandate = appInterface.createMandate( @@ -237,7 +237,7 @@ async def update_mandate( try: logger.debug(f"Updating mandate {mandateId} with data: {mandateData}") - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -280,7 +280,7 @@ async def delete_mandate( MULTI-TENANT: SysAdmin-only. """ try: - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -347,7 +347,7 @@ async def listMandateUsers( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -518,7 +518,7 @@ async def addUserToMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # 3. Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -627,7 +627,7 @@ async def removeUserFromMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -707,7 +707,7 @@ async def updateUserRolesInMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Get user's membership membership = rootInterface.getUserMandate(targetUserId, targetMandateId) @@ -810,7 +810,7 @@ def _hasMandateAdminRole(context: RequestContext, mandateId: str) -> bool: return False try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() for roleId in context.roleIds: roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index d3736b75..48902e66 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -10,7 +10,7 @@ import json from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelUtils import Prompt from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict @@ -58,7 +58,7 @@ async def get_prompts( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllPrompts(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -89,7 +89,7 @@ async def create_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Create a new prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Create prompt newPrompt = managementInterface.createPrompt(prompt) @@ -104,7 +104,7 @@ async def get_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Get a specific prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get prompt prompt = managementInterface.getPrompt(promptId) @@ -125,7 +125,7 @@ async def update_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Update an existing prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the prompt exists existingPrompt = managementInterface.getPrompt(promptId) @@ -160,7 +160,7 @@ async def delete_prompt( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the prompt exists existingPrompt = managementInterface.getPrompt(promptId) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 62a9b344..dab8bde9 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -17,7 +17,7 @@ import logging import json # Import interfaces and models -import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects +import modules.interfaces.interfaceDbApp as interfaceDbApp from modules.auth import limiter, getRequestContext, RequestContext # Import the attribute definition and helper functions @@ -179,7 +179,7 @@ async def get_users( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # MULTI-TENANT: Use mandateId from context (header) # SysAdmin without mandateId can see all users @@ -278,7 +278,7 @@ async def get_user( MULTI-TENANT: User must be in the same mandate (via UserMandate) or caller is SysAdmin. """ try: - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get user without filtering by enabled status user = appInterface.getUser(userId) @@ -333,7 +333,7 @@ async def create_user( Create a new user. MULTI-TENANT: User is created and automatically added to the current mandate. """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Extract fields from request model and call createUser with individual parameters from modules.datamodels.datamodelUam import AuthAuthority @@ -375,7 +375,7 @@ async def update_user( Update an existing user. MULTI-TENANT: Can only update users in the same mandate (unless SysAdmin). """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) @@ -430,7 +430,7 @@ async def reset_user_password( ) # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get target user target_user = appInterface.getUser(userId) @@ -525,7 +525,7 @@ async def change_password( """ try: # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Verify current password if not appInterface.verifyPassword(currentPassword, context.user.passwordHash): @@ -612,10 +612,10 @@ async def sendPasswordLink( """ try: from modules.shared.configuration import APP_CONFIG - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get target user targetUser = appInterface.getUser(userId) @@ -742,7 +742,7 @@ async def delete_user( Delete a user. MULTI-TENANT: Can only delete users in the same mandate (unless SysAdmin). """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeDataWorkflows.py similarity index 99% rename from modules/routes/routeWorkflows.py rename to modules/routes/routeDataWorkflows.py index ab9e1ff6..d3b2d825 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeDataWorkflows.py @@ -14,8 +14,8 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot -from modules.interfaces.interfaceDbChatbot import getInterface +import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.interfaces.interfaceDbChat import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models @@ -45,7 +45,7 @@ router = APIRouter( ) def getServiceChat(currentUser: User): - return interfaceDbChatbot.getInterface(currentUser) + return interfaceDbChat.getInterface(currentUser) # Consolidated endpoint for getting all workflows @router.get("/", response_model=PaginatedResponse[ChatWorkflow]) diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index ed1fd9f3..5be544a8 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum @@ -32,7 +32,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChat.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Workflow start endpoint @router.post("/start", response_model=ChatWorkflow) diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index b5b80e2e..977158f0 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -18,7 +18,7 @@ from modules.shared.timeUtils import parseTimestamp from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models @@ -43,7 +43,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChat.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Chatbot streaming endpoint (SSE) @router.post("/start/stream") diff --git a/modules/routes/routeFeatureRealEstate.py b/modules/routes/routeFeatureRealEstate.py index 7e130c1b..73364345 100644 --- a/modules/routes/routeFeatureRealEstate.py +++ b/modules/routes/routeFeatureRealEstate.py @@ -26,7 +26,7 @@ from modules.datamodels.datamodelRealEstate import ( ) # Import interfaces -from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands from modules.features.realEstate.mainRealEstate import ( diff --git a/modules/routes/routeFeatureTrustee.py b/modules/routes/routeFeatureTrustee.py index ad842db8..14b0a9e8 100644 --- a/modules/routes/routeFeatureTrustee.py +++ b/modules/routes/routeFeatureTrustee.py @@ -19,7 +19,7 @@ import io from modules.auth import limiter, getRequestContext, RequestContext from modules.interfaces.interfaceDbTrustee import getInterface -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface from modules.datamodels.datamodelTrustee import ( TrusteeOrganisation, diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index 53375849..f99dcd77 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -21,7 +21,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp from modules.shared.auditLogger import audit_logger diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index f649d2b2..3b059f9c 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, getCurrentUser from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelInvitation import Invitation -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index dae6e04e..953dd5f2 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -11,7 +11,7 @@ from modules.auth import limiter, getCurrentUser, getRequestContext, RequestCont from modules.datamodels.datamodelRbac import Role # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelMessaging import ( MessagingSubscription, MessagingSubscriptionRegistration, @@ -55,7 +55,7 @@ async def getSubscriptions( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllSubscriptions(pagination=paginationParams) if paginationParams: @@ -85,7 +85,7 @@ async def createSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Create a new subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) subscriptionData = subscription.model_dump(exclude={"id"}) newSubscription = managementInterface.createSubscription(subscriptionData) @@ -101,7 +101,7 @@ async def getSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Get a specific subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) subscription = managementInterface.getSubscription(subscriptionId) if not subscription: @@ -122,7 +122,7 @@ async def updateSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Update an existing subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingSubscription = managementInterface.getSubscription(subscriptionId) if not existingSubscription: @@ -151,7 +151,7 @@ async def deleteSubscription( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingSubscription = managementInterface.getSubscription(subscriptionId) if not existingSubscription: @@ -192,7 +192,7 @@ async def getSubscriptionRegistrations( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllRegistrations( subscriptionId=subscriptionId, pagination=paginationParams @@ -227,7 +227,7 @@ async def subscribeUser( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscriptionRegistration: """Subscribe user to a subscription with a specific channel""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) registration = managementInterface.subscribeUser( subscriptionId=subscriptionId, @@ -248,7 +248,7 @@ async def unsubscribeUser( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Unsubscribe user from a subscription for a specific channel""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) success = managementInterface.unsubscribeUser( subscriptionId=subscriptionId, @@ -284,7 +284,7 @@ async def getMyRegistrations( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllRegistrations( userId=currentUser.id, pagination=paginationParams @@ -318,7 +318,7 @@ async def updateRegistration( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscriptionRegistration: """Update a registration""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingRegistration = managementInterface.getRegistration(registrationId) if not existingRegistration: @@ -347,7 +347,7 @@ async def deleteRegistration( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a registration""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingRegistration = managementInterface.getRegistration(registrationId) if not existingRegistration: @@ -417,7 +417,7 @@ def _hasTriggerPermission(context: RequestContext) -> bool: return False try: - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() for roleId in context.roleIds: @@ -458,7 +458,7 @@ async def getDeliveries( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getDeliveries( subscriptionId=subscriptionId, userId=currentUser.id, # Users can only see their own deliveries @@ -492,7 +492,7 @@ async def getDelivery( currentUser: User = Depends(getCurrentUser) ) -> MessagingDelivery: """Get a specific delivery""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) delivery = managementInterface.getDelivery(deliveryId) if not delivery: diff --git a/modules/routes/routeSecurityAdmin.py b/modules/routes/routeSecurityAdmin.py index 3ba35e65..36388c9f 100644 --- a/modules/routes/routeSecurityAdmin.py +++ b/modules/routes/routeSecurityAdmin.py @@ -13,7 +13,7 @@ import logging from modules.auth import getCurrentUser, limiter, requireSysAdmin from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 2a8f65fd..9642df00 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -13,7 +13,7 @@ from requests_oauthlib import OAuth2Session import httpx from modules.shared.configuration import APP_CONFIG -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection from modules.auth import getCurrentUser, limiter from modules.auth import createAccessToken, setAccessTokenCookie, createRefreshToken, setRefreshTokenCookie diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 96f22136..64984eef 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -16,7 +16,7 @@ from jose import jwt # Import auth modules from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 77dc9885..6d034607 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -13,7 +13,7 @@ import msal import httpx from modules.shared.configuration import APP_CONFIG -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter diff --git a/modules/routes/routeSharepoint.py b/modules/routes/routeSharepoint.py index 7915b0f1..aa62afc6 100644 --- a/modules/routes/routeSharepoint.py +++ b/modules/routes/routeSharepoint.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Query, Request, sta from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User, UserConnection -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.interfaces.interfaceDbApp import getInterface from modules.services import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index fb4e6512..ff91dc6b 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -49,13 +49,13 @@ class Services: # Initialize interfaces with explicit mandateId - from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbTrustee import getInterface as getTrusteeInterface diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 13739dea..199201eb 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -65,7 +65,7 @@ class ExtractionService: results: List[ContentExtracted] = [] # Lazy import to avoid circular deps and heavy init at module import - from modules.interfaces.interfaceDbComponentObjects import getInterface + from modules.interfaces.interfaceDbManagement import getInterface dbInterface = getInterface() totalDocs = len(documents) diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 5d3a8497..2c975f1d 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -157,11 +157,11 @@ class UtilsService: def storeDebugMessageAndDocuments(self, message, currentUser): """ - Wrapper to store debug messages and documents via interfaceDbChatbot. - Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatbot. + Wrapper to store debug messages and documents via interfaceDbChat. + Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChat. """ try: - from modules.interfaces.interfaceDbChatbot import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index a6e3a78a..136dd2cb 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -452,10 +452,10 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - from modules.interfaces.interfaceDbAppObjects import getRootInterface + import modules.interfaces.interfaceDbChat as interfaceDbChat + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() - interfaceDbChat = interfaceDbChatbot.getInterface(rootInterface.currentUser) + interfaceDbChat = interfaceDbChat.getInterface(rootInterface.currentUser) # Get workflow logs chatData = interfaceDbChat.getUnifiedChatData(context.workflow.id, None) diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index 05a3f34b..f807fe24 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -27,7 +27,7 @@ class MethodAiOperationsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -94,8 +94,8 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) # Import and initialize services from modules.services import getInterface as getServices @@ -201,8 +201,8 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,8 +283,8 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 9da22d66..3e46bc0c 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -27,7 +27,7 @@ from modules.datamodels.datamodelWorkflow import AiResponse class AIBehaviorTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -45,7 +45,7 @@ class AIBehaviorTester: from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + import modules.interfaces.interfaceDbChat as interfaceDbChat currentTimestamp = time.time() @@ -67,7 +67,7 @@ class AIBehaviorTester: ) # SAVE workflow to database so it exists for access control - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 7beaeec4..8850ae2b 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -23,13 +23,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class WorkflowWithDocumentsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -192,7 +192,7 @@ class WorkflowWithDocumentsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) if not currentWorkflow: @@ -225,7 +225,7 @@ class WorkflowWithDocumentsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 698c9698..106e2999 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -25,13 +25,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class WorkflowPromptVariationsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -115,7 +115,7 @@ class WorkflowPromptVariationsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(workflow.id) if not currentWorkflow: @@ -140,7 +140,7 @@ class WorkflowPromptVariationsTester: def _analyzeWorkflowResults(self, workflow: Any) -> Dict[str, Any]: """Analyze workflow results and extract information.""" - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(workflow.id) if not workflow: diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index d9c4d9b8..3c85460e 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -24,13 +24,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class DocumentGenerationFormatsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -251,7 +251,7 @@ class DocumentGenerationFormatsTester: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -296,7 +296,7 @@ class DocumentGenerationFormatsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 45a364ce..19bdb12f 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -24,13 +24,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class DocumentGenerationFormatsTester10: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -248,7 +248,7 @@ class DocumentGenerationFormatsTester10: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -293,7 +293,7 @@ class DocumentGenerationFormatsTester10: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 6d1735ad..64c8f93e 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -26,13 +26,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class CodeGenerationFormatsTester11: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -190,7 +190,7 @@ class CodeGenerationFormatsTester11: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -235,7 +235,7 @@ class CodeGenerationFormatsTester11: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/integration/options/test_options_api.py b/tests/integration/options/test_options_api.py index 68a276e2..7007bf2a 100644 --- a/tests/integration/options/test_options_api.py +++ b/tests/integration/options/test_options_api.py @@ -9,7 +9,7 @@ import pytest import secrets from fastapi.testclient import TestClient from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface @pytest.fixture