From 5c0ab3f8935dfdfacb8735fdcaddf5a428c19680 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 17 Jan 2026 02:17:58 +0100 Subject: [PATCH 01/32] implemented multimandate --- app.py | 24 +- env_dev.env | 32 +- env_int.env | 32 +- env_prod.env | 32 +- modules/aicore/aicorePluginTavily.py | 3 +- modules/auth/__init__.py | 24 +- modules/auth/authentication.py | 206 ++- modules/connectors/connectorDbPostgre.py | 6 +- modules/datamodels/datamodelFeatures.py | 83 ++ modules/datamodels/datamodelInvitation.py | 120 ++ modules/datamodels/datamodelMembership.py | 150 ++ modules/datamodels/datamodelRbac.py | 174 ++- modules/datamodels/datamodelSecurity.py | 72 +- modules/datamodels/datamodelUam.py | 184 ++- modules/features/chatbot/mainChatbot.py | 15 +- .../dynamicOptions/mainDynamicOptions.py | 2 +- .../mainNeutralizePlayground.py | 7 +- modules/features/realEstate/mainRealEstate.py | 53 +- modules/features/workflow/mainWorkflow.py | 24 +- modules/interfaces/interfaceBootstrap.py | 1283 ++++++----------- modules/interfaces/interfaceDbAppObjects.py | 475 +++++- modules/interfaces/interfaceDbChatObjects.py | 89 +- .../interfaces/interfaceDbComponentObjects.py | 45 +- .../interfaceDbRealEstateObjects.py | 82 +- .../interfaces/interfaceDbTrusteeObjects.py | 235 +-- modules/interfaces/interfaceFeatures.py | 478 ++++++ modules/interfaces/interfaceRbac.py | 46 +- modules/interfaces/interfaceVoiceObjects.py | 32 +- modules/routes/routeAdminRbacRoles.py | 465 +++--- modules/routes/routeDataAutomation.py | 6 +- modules/routes/routeDataMandates.py | 548 ++++++- modules/routes/routeDataUsers.py | 280 +++- ...ionEvents.py => routeFeatureAutomation.py} | 32 +- ...ayground.py => routeFeatureChatDynamic.py} | 19 +- ...routeChatbot.py => routeFeatureChatbot.py} | 28 +- ...ation.py => routeFeatureNeutralization.py} | 43 +- ...ealEstate.py => routeFeatureRealEstate.py} | 141 +- ...eDataTrustee.py => routeFeatureTrustee.py} | 237 ++- modules/routes/routeFeatures.py | 625 ++++++++ modules/routes/routeGdpr.py | 514 +++++++ modules/routes/routeInvitations.py | 812 +++++++++++ modules/routes/routeMessaging.py | 53 +- modules/routes/routeRbac.py | 275 +--- modules/routes/routeRbacExport.py | 608 ++++++++ modules/routes/routeSecurityAdmin.py | 376 ++--- modules/routes/routeSecurityGoogle.py | 11 +- modules/routes/routeSecurityLocal.py | 21 +- modules/routes/routeSecurityMsft.py | 11 +- modules/security/rbac.py | 387 ++++- modules/security/rootAccess.py | 49 +- modules/services/__init__.py | 27 +- modules/shared/dbMultiTenantOptimizations.py | 438 ++++++ modules/workflows/workflowManager.py | 2 +- 53 files changed, 7558 insertions(+), 2458 deletions(-) create mode 100644 modules/datamodels/datamodelFeatures.py create mode 100644 modules/datamodels/datamodelInvitation.py create mode 100644 modules/datamodels/datamodelMembership.py create mode 100644 modules/interfaces/interfaceFeatures.py rename modules/routes/{routeAdminAutomationEvents.py => routeFeatureAutomation.py} (85%) rename modules/routes/{routeChatPlayground.py => routeFeatureChatDynamic.py} (86%) rename modules/routes/{routeChatbot.py => routeFeatureChatbot.py} (96%) rename modules/routes/{routeDataNeutralization.py => routeFeatureNeutralization.py} (85%) rename modules/routes/{routeRealEstate.py => routeFeatureRealEstate.py} (91%) rename modules/routes/{routeDataTrustee.py => routeFeatureTrustee.py} (79%) create mode 100644 modules/routes/routeFeatures.py create mode 100644 modules/routes/routeGdpr.py create mode 100644 modules/routes/routeInvitations.py create mode 100644 modules/routes/routeRbacExport.py create mode 100644 modules/shared/dbMultiTenantOptimizations.py diff --git a/app.py b/app.py index 8ddd806e..7ed57ed9 100644 --- a/app.py +++ b/app.py @@ -405,7 +405,7 @@ app.include_router(userRouter) from modules.routes.routeDataFiles import router as fileRouter app.include_router(fileRouter) -from modules.routes.routeDataNeutralization import router as neutralizationRouter +from modules.routes.routeFeatureNeutralization import router as neutralizationRouter app.include_router(neutralizationRouter) from modules.routes.routeDataPrompts import router as promptRouter @@ -417,10 +417,10 @@ app.include_router(connectionsRouter) from modules.routes.routeWorkflows import router as workflowRouter app.include_router(workflowRouter) -from modules.routes.routeChatPlayground import router as chatPlaygroundRouter +from modules.routes.routeFeatureChatDynamic import router as chatPlaygroundRouter app.include_router(chatPlaygroundRouter) -from modules.routes.routeRealEstate import router as realEstateRouter +from modules.routes.routeFeatureRealEstate import router as realEstateRouter app.include_router(realEstateRouter) from modules.routes.routeSecurityLocal import router as localRouter @@ -444,7 +444,7 @@ app.include_router(sharepointRouter) from modules.routes.routeDataAutomation import router as automationRouter app.include_router(automationRouter) -from modules.routes.routeAdminAutomationEvents import router as adminAutomationEventsRouter +from modules.routes.routeFeatureAutomation import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) from modules.routes.routeRbac import router as rbacRouter @@ -456,9 +456,21 @@ app.include_router(optionsRouter) from modules.routes.routeMessaging import router as messagingRouter app.include_router(messagingRouter) -from modules.routes.routeChatbot import router as chatbotRouter +from modules.routes.routeFeatureChatbot import router as chatbotRouter app.include_router(chatbotRouter) -from modules.routes.routeDataTrustee import router as trusteeRouter +from modules.routes.routeFeatureTrustee import router as trusteeRouter app.include_router(trusteeRouter) +# Phase 8: New Feature Routes +from modules.routes.routeFeatures import router as featuresRouter +app.include_router(featuresRouter) + +from modules.routes.routeInvitations import router as invitationsRouter +app.include_router(invitationsRouter) + +from modules.routes.routeRbacExport import router as rbacExportRouter +app.include_router(rbacExportRouter) + +from modules.routes.routeGdpr import router as gdprRouter +app.include_router(gdprRouter) diff --git a/env_dev.env b/env_dev.env index 19d718d1..93523018 100644 --- a/env_dev.env +++ b/env_dev.env @@ -8,33 +8,11 @@ APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/key.txt APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9 APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9 -# PostgreSQL Storage (new) -DB_APP_HOST=localhost -DB_APP_DATABASE=poweron_app -DB_APP_USER=poweron_dev -DB_APP_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEcUIxNEFfQ2xnS0RrSC1KNnUxTlVvTGZoMHgzaEI4Z3NlVzVROTVLak5Ubi1vaEZubFZaMTFKMGd6MXAxekN2d2NvMy1hRjg2UVhybktlcFA5anZ1WjFlQmZhcXdwaGhWdzRDc3ExeUhzWTg9 -DB_APP_PORT=5432 - -# PostgreSQL Storage (new) -DB_CHAT_HOST=localhost -DB_CHAT_DATABASE=poweron_chat -DB_CHAT_USER=poweron_dev -DB_CHAT_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERFNzNVhoalpCR0QxYXAwdEpXWXVVOTdZdWtqWW5FNXFGcFl2amNYLWYwYl9STXltRlFxLWNzVWlMVnNYdXk0RklnRExFT0FaQjg2aGswNnhhSGhCN29KN2VEb2FlUV9NTlV3b0tLelplSVU9 -DB_CHAT_PORT=5432 - -# PostgreSQL Storage (new) -DB_MANAGEMENT_HOST=localhost -DB_MANAGEMENT_DATABASE=poweron_management -DB_MANAGEMENT_USER=poweron_dev -DB_MANAGEMENT_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEUldqSTVpUnFqdGhITDYzT3RScGlMYVdTMmZhOXdudDRCc3dhdllOd3l6MS1vWHY2MjVsTUF1Sk9saEJOSk9ONUlBZjQwb2c2T1gtWWJhcXFzVVVXd01xc0U0b0lJX0JyVDRxaDhNS01JcWs9 -DB_MANAGEMENT_PORT=5432 - -# PostgreSQL Storage (new) -DB_REALESTATE_HOST=localhost -DB_REALESTATE_DATABASE=poweron_realestate -DB_REALESTATE_USER=poweron_dev -DB_REALESTATE_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEcUIxNEFfQ2xnS0RrSC1KNnUxTlVvTGZoMHgzaEI4Z3NlVzVROTVLak5Ubi1vaEZubFZaMTFKMGd6MXAxekN2d2NvMy1hRjg2UVhybktlcFA5anZ1WjFlQmZhcXdwaGhWdzRDc3ExeUhzWTg9 -DB_REALESTATE_PORT=5432 +# PostgreSQL DB Host +DB_HOST=localhost +DB_USER=poweron_dev +DB_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEcUIxNEFfQ2xnS0RrSC1KNnUxTlVvTGZoMHgzaEI4Z3NlVzVROTVLak5Ubi1vaEZubFZaMTFKMGd6MXAxekN2d2NvMy1hRjg2UVhybktlcFA5anZ1WjFlQmZhcXdwaGhWdzRDc3ExeUhzWTg9 +DB_PORT=5432 # Security Configuration APP_JWT_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERjlrSktmZHVuQnJ1VVJDdndLaUcxZGJsT2ZlUFRlcFdOZ001RnlzM2FhLWhRV2tjWWFhaWQwQ3hkcUFvbThMcndxSjFpYTdfRV9OZGhTcksxbXFTZWg5MDZvOHpCVXBHcDJYaHlJM0tyNWRZckZsVHpQcmxTZHJoZUs1M3lfU2ljRnJaTmNSQ0w0X085OXI0QW80M2xfQnJqZmZ6VEh3TUltX0xzeE42SGtZPQ== diff --git a/env_int.env b/env_int.env index 7047dc6e..05313802 100644 --- a/env_int.env +++ b/env_int.env @@ -8,33 +8,11 @@ APP_KEY_SYSVAR = CONFIG_KEY APP_INIT_PASS_ADMIN_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjWm41MWZ4TUZGaVlrX3pWZWNwakJsY3Facm0wLVZDd1VKeTFoZEVZQnItcEdUUnVJS1NXeDBpM2xKbGRsYmxOSmRhc29PZjJSU2txQjdLbUVrTTE1NEJjUXBHbV9NOVJWZUR3QlJkQnJvTEU9 APP_INIT_PASS_EVENT_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjdmtrakgxa0djekZVNGtTZV8wM2I5UUpCZllveVBMWXROYk5yS3BiV3JEelJSM09VYTRONHpnY3VtMGxDRk5JTEZSRFhtcDZ0RVRmZ1RicTFhb3c5dVZRQ1o4SmlkLVpPTW5MMTU2eTQ0Vkk9 -# PostgreSQL Storage (new) -DB_APP_HOST=gateway-int-server.postgres.database.azure.com -DB_APP_DATABASE=poweron_app -DB_APP_USER=heeshkdlby -DB_APP_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjb2dka2pnN0tUbW1EU0w1Rk1jNERKQ0Z1U3JkVDhuZWZDM0g5M0kwVDE5VHdubkZna3gtZVAxTnl4MDdrR1c1ZXJ3ejJHYkZvcGUwbHJaajBGOWJob0EzRXVHc0JnZkJyNGhHZTZHOXBxd2c9 -DB_APP_PORT=5432 - -# PostgreSQL Storage (new) -DB_CHAT_HOST=gateway-int-server.postgres.database.azure.com -DB_CHAT_DATABASE=poweron_chat -DB_CHAT_USER=heeshkdlby -DB_CHAT_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjczYzOUtTa21MMGJVTUQ5UmFfdWc3YlhCbWZOeXFaNEE1QzdJV3BLVjhnalBkLVVCMm5BZzdxdlFXQXc2RHYzLWtPSFZkZE1iWG9rQ1NkVWlpRnF5TURVbnl1cm9iYXlSMGYxd1BGYVc0VDA9 -DB_CHAT_PORT=5432 - -# PostgreSQL Storage (new) -DB_MANAGEMENT_HOST=gateway-int-server.postgres.database.azure.com -DB_MANAGEMENT_DATABASE=poweron_management -DB_MANAGEMENT_USER=heeshkdlby -DB_MANAGEMENT_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjTnJKNlJMNmEwQ0Y5dVNrR3pkZk9SQXVvLTRTNW9lQ1g3TTE5cFhBNTd5UENqWW9qdWd3NWNseWhnUHJveDJyd1Z3X1czS3VuZnAwZHBXYVNQWlZsRy12ME42NndEVlR5X3ZPdFBNNmhLYm89 -DB_MANAGEMENT_PORT=5432 - -# PostgreSQL Storage (new) -DB_REALESTATE_HOST=localhost -DB_REALESTATE_DATABASE=poweron_realestate -DB_REALESTATE_USER=poweron_dev -DB_REALESTATE_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjTnJKNlJMNmEwQ0Y5dVNrR3pkZk9SQXVvLTRTNW9lQ1g3TTE5cFhBNTd5UENqWW9qdWd3NWNseWhnUHJveDJyd1Z3X1czS3VuZnAwZHBXYVNQWlZsRy12ME42NndEVlR5X3ZPdFBNNmhLYm89 -DB_REALESTATE_PORT=5432 +# PostgreSQL DB Host +DB_HOST=gateway-int-server.postgres.database.azure.com +DB_USER=heeshkdlby +DB_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjczYzOUtTa21MMGJVTUQ5UmFfdWc3YlhCbWZOeXFaNEE1QzdJV3BLVjhnalBkLVVCMm5BZzdxdlFXQXc2RHYzLWtPSFZkZE1iWG9rQ1NkVWlpRnF5TURVbnl1cm9iYXlSMGYxd1BGYVc0VDA9 +DB_PORT=5432 # Security Configuration APP_JWT_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjNUctb2RwU25iR3ZnanBOdHZhWUtIajZ1RnZzTEp4aDR0MktWRjNoeVBrY1Npd1R0VE9YVHp3M2w1cXRzbUxNaU82QUJvaDNFeVQyN05KblRWblBvbWtoT0VXbkNBbDQ5OHhwSUFnaDZGRG10Vmgtdm1YUkRsYUhFMzRVZURmSFlDTFIzVWg4MXNueDZyMGc5aVpFdWRxY3dkTExGM093ZTVUZVl5LUhGWnlRPQ== diff --git a/env_prod.env b/env_prod.env index 32cfe3ad..57a4e83c 100644 --- a/env_prod.env +++ b/env_prod.env @@ -8,33 +8,11 @@ APP_INIT_PASS_ADMIN_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3UnJRV0sySFlDblpXUlREclREaW APP_INIT_PASS_EVENT_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3QVpIY19DQVZSSzJmc2F0VEZvQlU1cHBhTEgxdHdnR3g4eW01aTEzYTUxc1gxTDR1RVVpSHRXYjV6N1BLZUdCUGlfOW1qdy0xSHFVRkNBcGZvaGlSSkZycXRuUllaWnpyVGRoeFg1dGEyNUk9 APP_API_URL = https://gateway-prod.poweron-center.net -# PostgreSQL Storage (new) -DB_APP_HOST=gateway-prod-server.postgres.database.azure.com -DB_APP_DATABASE=poweron_app -DB_APP_USER=gzxxmcrdhn -DB_APP_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3cm5LQWV1OURQanVyTklVaVhJbDI2Y1Itb29pTWFmR2RYM0pyYUhhRUpWZ29tWWwzSmdQeVhScHlHQWVyY0xUTElIdVBJUjh5Zm9ZMzg1ZERNQXZ6TXlGb2tYOGpDX1gzXzB3UUlCM1ZaYWM9 -DB_APP_PORT=5432 - -# PostgreSQL Storage (new) -DB_CHAT_HOST=gateway-prod-server.postgres.database.azure.com -DB_CHAT_DATABASE=poweron_chat -DB_CHAT_USER=gzxxmcrdhn -DB_CHAT_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3Y1JScGxjZG9TdUkwaHRzSHZhRHpNcDV3N1U2TnIwZ21PRG5TWFFfR1k0N3BiRk5WelVadjlnXzVSTDZ6NXFQNFpqbnJ1R3dNVkJocm1zVEgtSk0xaDRiR19zNDBEbVIzSk51ekNlQ0Z3b0U9 -DB_CHAT_PORT=5432 - -# PostgreSQL Storage (new) -DB_MANAGEMENT_HOST=gateway-prod-server.postgres.database.azure.com -DB_MANAGEMENT_DATABASE=poweron_management -DB_MANAGEMENT_USER=gzxxmcrdhn -DB_MANAGEMENT_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3ZWpySThqdlVmWWd5dGxmWE91RVBsenZrQmNhSzVxbktmYzZ1RlM3cXhTMUdXRV9wX1lfLTJXLTFzeUo0R3pWLXlmUWdrZ2x6QkFlZVRXaEF6aUdRbDlzb1FfcWtub0dxSGp3OVVQWGg3enM9 -DB_MANAGEMENT_PORT=5432 - -# PostgreSQL Storage (new) -DB_REALESTATE_HOST=localhost -DB_REALESTATE_DATABASE=poweron_realestate -DB_REALESTATE_USER=poweron_dev -DB_REALESTATE_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3ZWpySThqdlVmWWd5dGxmWE91RVBsenZrQmNhSzVxbktmYzZ1RlM3cXhTMUdXRV9wX1lfLTJXLTFzeUo0R3pWLXlmUWdrZ2x6QkFlZVRXaEF6aUdRbDlzb1FfcWtub0dxSGp3OVVQWGg3enM9 -DB_REALESTATE_PORT=5432 +# PostgreSQL DB Host +DB_HOST=gateway-prod-server.postgres.database.azure.com +DB_USER=gzxxmcrdhn +DB_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3Y1JScGxjZG9TdUkwaHRzSHZhRHpNcDV3N1U2TnIwZ21PRG5TWFFfR1k0N3BiRk5WelVadjlnXzVSTDZ6NXFQNFpqbnJ1R3dNVkJocm1zVEgtSk0xaDRiR19zNDBEbVIzSk51ekNlQ0Z3b0U9 +DB_PORT=5432 # Security Configuration APP_JWT_KEY_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3elhfV0Rnd2pQRjlMdkVwX1FnSmRhSzNZUlV5SVpaWXBNX1hpa2xPZGdMSWpnN2ZINHQxeGZnNHJweU5pZjlyYlY5Qm9zOUZEbl9wUEgtZHZXd1NhR19JSG9kbFU4MnFGQnllbFhRQVphRGQyNHlFVWR5VHQyUUpqN0stUmRuY2QyTi1oalczRHpLTEJqWURjZWs4YjZvT2U5YnFqcXEwdEpxV05fX05QMmtrPQ== diff --git a/modules/aicore/aicorePluginTavily.py b/modules/aicore/aicorePluginTavily.py index d66c3005..1a5cbc65 100644 --- a/modules/aicore/aicorePluginTavily.py +++ b/modules/aicore/aicorePluginTavily.py @@ -728,8 +728,7 @@ class AiTavily(BaseConnectorAi): maxBreadth=webCrawlPrompt.maxWidth or 40 # Use same as limit for breadth ) - # If we got multiple pages from the crawl, we need to format them differently - # Return the first result for backwards compatibility, but include total page count + # Format multiple pages from the crawl into a single response if crawlResults and len(crawlResults) > 0: # Get all pages content with error handling allContent = "" diff --git a/modules/auth/__init__.py b/modules/auth/__init__.py index b375ef15..49024b66 100644 --- a/modules/auth/__init__.py +++ b/modules/auth/__init__.py @@ -3,9 +3,23 @@ """ Authentication and authorization modules for routes and services. High-level security functionality that depends on FastAPI and interfaces. + +Multi-Tenant Design: +- RequestContext: Per-request context with user, mandate, feature instance, roles +- getRequestContext: FastAPI dependency to extract context from X-Mandate-Id header +- requireSysAdmin: FastAPI dependency for system-level admin operations """ -from .authentication import getCurrentUser, limiter, SECRET_KEY, ALGORITHM, cookieAuth +from .authentication import ( + getCurrentUser, + limiter, + SECRET_KEY, + ALGORITHM, + cookieAuth, + RequestContext, + getRequestContext, + requireSysAdmin, +) from .jwtService import ( createAccessToken, createRefreshToken, @@ -20,22 +34,30 @@ from .tokenRefreshMiddleware import TokenRefreshMiddleware, ProactiveTokenRefres from .csrf import CSRFMiddleware __all__ = [ + # Authentication "getCurrentUser", "limiter", "SECRET_KEY", "ALGORITHM", "cookieAuth", + # Multi-Tenant Context + "RequestContext", + "getRequestContext", + "requireSysAdmin", + # JWT Service "createAccessToken", "createRefreshToken", "setAccessTokenCookie", "setRefreshTokenCookie", "clearAccessTokenCookie", "clearRefreshTokenCookie", + # Token Management "TokenManager", "token_refresh_service", "TokenRefreshService", "TokenRefreshMiddleware", "ProactiveTokenRefreshMiddleware", + # CSRF "CSRFMiddleware", ] diff --git a/modules/auth/authentication.py b/modules/auth/authentication.py index 2759a07a..f6cf0f0d 100644 --- a/modules/auth/authentication.py +++ b/modules/auth/authentication.py @@ -3,10 +3,16 @@ """ Authentication module for backend API. Handles JWT-based authentication, token generation, and user context. + +Multi-Tenant Design: +- Token ist NICHT an einen Mandanten gebunden +- User arbeitet parallel in mehreren Mandanten (z.B. mehrere Browser-Tabs) +- Mandant-Kontext wird per Request-Header (X-Mandate-Id) bestimmt +- Request-Context kapselt User + Mandant + Feature-Instanz + geladene Rollen """ -from typing import Optional, Dict, Any, Tuple -from fastapi import Depends, HTTPException, status, Request, Response +from typing import Optional, Dict, Any, Tuple, List +from fastapi import Depends, HTTPException, status, Request, Response, Header from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from jose import JWTError, jwt import logging @@ -15,9 +21,10 @@ from slowapi.util import get_remote_address from modules.shared.configuration import APP_CONFIG from modules.security.rootAccess import getRootDbAppConnector, getRootUser -from modules.interfaces.interfaceDbAppObjects import getInterface -from modules.datamodels.datamodelUam import User, AuthAuthority +from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.datamodels.datamodelUam import User, AuthAuthority, AccessLevel from modules.datamodels.datamodelSecurity import Token +from modules.datamodels.datamodelRbac import AccessRule # Get Config Data SECRET_KEY = APP_CONFIG.get("APP_JWT_KEY_SECRET") @@ -98,15 +105,16 @@ def _getUserBase(token: str = Depends(cookieAuth)) -> User: if username is None: raise credentialsException - # Extract mandate ID and user ID from token - mandateId: str = payload.get("mandateId") + # Extract user ID from token + # MULTI-TENANT: mandateId is NO LONGER in the token - it comes from X-Mandate-Id header userId: str = payload.get("userId") authority: str = payload.get("authenticationAuthority") tokenId: Optional[str] = payload.get("jti") sessionId: Optional[str] = payload.get("sid") or payload.get("sessionId") - if not mandateId or not userId: - logger.error(f"Missing context in token: mandateId={mandateId}, userId={userId}") + # Only userId is required in token now (no mandateId) + if not userId: + logger.error(f"Missing userId in token") raise credentialsException except JWTError: @@ -129,9 +137,10 @@ def _getUserBase(token: str = Depends(cookieAuth)) -> User: logger.warning(f"User {username} is disabled") raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="User is disabled") - # Ensure the user has the correct context - if str(user.mandateId) != str(mandateId) or str(user.id) != str(userId): - logger.error(f"User context mismatch: token(mandateId={mandateId}, userId={userId}) vs user(mandateId={user.mandateId}, id={user.id})") + # Ensure the user ID in token matches the user in database + # MULTI-TENANT: mandateId is NO LONGER checked here - it comes from headers + if str(user.id) != str(userId): + logger.error(f"User ID mismatch: token(userId={userId}) vs user(id={user.id})") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="User context has changed. Please log in again.", @@ -166,17 +175,18 @@ def _getUserBase(token: str = Depends(cookieAuth)) -> User: db_token = db_tokens[0] token_authority = str(db_token.get("authority", "")).lower() if token_authority == str(AuthAuthority.LOCAL.value): - # Must be active and match user/session/mandate + # Must be active and match user/session + # MULTI-TENANT: mandateId is NOT checked here - tokens are no longer mandate-bound active_token = appInterface.findActiveTokenById( tokenId=tokenId, userId=user.id, authority=AuthAuthority.LOCAL, sessionId=sessionId, - mandateId=str(mandateId) if mandateId else None, + mandateId=None, # Token is no longer mandate-bound ) if not active_token: logger.info( - f"Local JWT db record not active/valid: jti={tokenId}, userId={user.id}, mandateId={mandateId}, sessionId={sessionId}" + f"Local JWT db record not active/valid: jti={tokenId}, userId={user.id}, sessionId={sessionId}" ) raise credentialsException else: @@ -203,3 +213,171 @@ def getCurrentUser(currentUser: User = Depends(_getUserBase)) -> User: return currentUser +# ============================================================================= +# MULTI-TENANT: Request Context System +# ============================================================================= + +class RequestContext: + """ + Request context for multi-tenant operations. + + Contains user, mandate context, feature instance context, and loaded role IDs. + This context is per-request (not persisted) - follows stateless design. + + IMPORTANT: SysAdmin also needs explicit membership for mandate context! + isSysAdmin flag does NOT give implicit access to mandate data. + """ + + def __init__(self, user: User): + self.user: User = user + self.mandateId: Optional[str] = None + self.featureInstanceId: Optional[str] = None + self.roleIds: List[str] = [] + + # Request-scoped cache: rules loaded only once per request + self._cachedRules: Optional[List[tuple]] = None + + def getRules(self) -> List[tuple]: + """ + Loads rules once per request (not across requests). + Returns list of (priority, AccessRule) tuples. + """ + if self._cachedRules is None: + if not self.mandateId: + # No mandate context = no rules + self._cachedRules = [] + else: + try: + rootUser = getRootUser() + appInterface = getInterface(rootUser) + self._cachedRules = appInterface.rbac.getRulesForUserBulk( + self.user.id, + self.mandateId, + self.featureInstanceId + ) + except Exception as e: + logger.error(f"Error loading RBAC rules: {e}") + self._cachedRules = [] + return self._cachedRules + + @property + def isSysAdmin(self) -> bool: + """Convenience property to check if user is a system admin.""" + return getattr(self.user, 'isSysAdmin', False) + + +def getRequestContext( + request: Request, + mandateId: Optional[str] = Header(None, alias="X-Mandate-Id"), + featureInstanceId: Optional[str] = Header(None, alias="X-Instance-Id"), + currentUser: User = Depends(getCurrentUser) +) -> RequestContext: + """ + Determines request context from headers. + Checks authorization and loads role IDs. + + IMPORTANT: Even SysAdmin needs explicit membership for mandate context! + SysAdmin flag does NOT give implicit access to mandate data. + + Args: + request: FastAPI Request object + mandateId: Mandate ID from X-Mandate-Id header + featureInstanceId: Feature instance ID from X-Instance-Id header + currentUser: Current authenticated user + + Returns: + RequestContext with user, mandate, roles + + Raises: + HTTPException 403: If user is not member of mandate or has no feature access + """ + ctx = RequestContext(user=currentUser) + + # Get root interface for membership checks + rootInterface = getRootInterface() + + if mandateId: + # Check mandate membership - ALSO for SysAdmin! + # SysAdmin must be explicitly added to the mandate + membership = rootInterface.getUserMandate(currentUser.id, mandateId) + if not membership: + # No implicit access for SysAdmin - Fail-Fast! + logger.warning(f"User {currentUser.id} is not member of mandate {mandateId}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Not member of mandate" + ) + + if not membership.enabled: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate membership is disabled" + ) + + ctx.mandateId = mandateId + + # Load roles via Junction Table + ctx.roleIds = rootInterface.getRoleIdsForUserMandate(membership.id) + + if featureInstanceId: + # Check feature access - ALSO for SysAdmin! + access = rootInterface.getFeatureAccess(currentUser.id, featureInstanceId) + if not access: + logger.warning(f"User {currentUser.id} has no access to feature instance {featureInstanceId}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="No access to feature instance" + ) + + if not access.enabled: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Feature access is disabled" + ) + + ctx.featureInstanceId = featureInstanceId + + # Add instance roles + instanceRoleIds = rootInterface.getRoleIdsForFeatureAccess(access.id) + ctx.roleIds.extend(instanceRoleIds) + + return ctx + + +def requireSysAdmin(currentUser: User = Depends(getCurrentUser)) -> User: + """ + SysAdmin check for system-level operations. + + Use this dependency for endpoints that require SysAdmin privileges. + SysAdmin has access to system-level operations, but NOT to mandate data. + + Args: + currentUser: Current authenticated user + + Returns: + User if they are a SysAdmin + + Raises: + HTTPException 403: If user is not a SysAdmin + """ + if not getattr(currentUser, 'isSysAdmin', False): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="SysAdmin privileges required" + ) + + # Audit for all SysAdmin actions + try: + from modules.shared.auditLogger import audit_logger + audit_logger.logSecurityEvent( + userId=str(currentUser.id), + mandateId="system", + action="sysadmin_action", + details="System-level operation" + ) + except Exception: + # Don't fail if audit logging fails + pass + + return currentUser + diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index ba452891..5cf2dc62 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -638,14 +638,12 @@ class DatabaseConnector: # Only set _createdBy if userId is valid (not None or empty string) if self.userId: record["_createdBy"] = self.userId - else: - logger.warning(f"Attempting to create record with empty userId - _createdBy will not be set") + # No warning - empty userId is normal during bootstrap # Also ensure _createdBy is set even if _createdAt exists but _createdBy is missing/empty elif "_createdBy" not in record or not record.get("_createdBy"): if self.userId: record["_createdBy"] = self.userId - else: - logger.warning(f"Attempting to set _createdBy with empty userId for record {recordId}") + # No warning - empty userId is normal during bootstrap # Always update modification metadata record["_modifiedAt"] = currentTime if self.userId: diff --git a/modules/datamodels/datamodelFeatures.py b/modules/datamodels/datamodelFeatures.py new file mode 100644 index 00000000..e772a19e --- /dev/null +++ b/modules/datamodels/datamodelFeatures.py @@ -0,0 +1,83 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Feature models: Feature, FeatureInstance.""" + +import uuid +from typing import Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels + + +class Feature(BaseModel): + """ + Feature-Definition (global, z.B. 'trustee', 'chatbot'). + Features sind die verfügbaren Funktionalitäten der Plattform. + """ + code: str = Field( + description="Unique feature code (Primary Key), z.B. 'trustee', 'chatbot'", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} + ) + label: dict = Field( + default_factory=dict, + description="Feature label in multiple languages (I18n)", + json_schema_extra={"frontend_type": "multilingual", "frontend_readonly": False, "frontend_required": True} + ) + icon: str = Field( + default="", + description="Icon identifier for the feature", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False} + ) + + +registerModelLabels( + "Feature", + {"en": "Feature", "de": "Feature", "fr": "Fonctionnalité"}, + { + "code": {"en": "Code", "de": "Code", "fr": "Code"}, + "label": {"en": "Label", "de": "Bezeichnung", "fr": "Libellé"}, + "icon": {"en": "Icon", "de": "Symbol", "fr": "Icône"}, + }, +) + + +class FeatureInstance(BaseModel): + """ + Instanz eines Features in einem Mandanten. + Ein Mandant kann mehrere Instanzen desselben Features haben. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the feature instance", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureCode: str = Field( + description="FK → Feature.code", + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": True} + ) + mandateId: str = Field( + description="FK → Mandate.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + label: str = Field( + default="", + description="Instance label, z.B. 'Buchhaltung 2025'", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} + ) + enabled: bool = Field( + default=True, + description="Whether this feature instance is enabled", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) + + +registerModelLabels( + "FeatureInstance", + {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de fonctionnalité"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "featureCode": {"en": "Feature", "de": "Feature", "fr": "Fonctionnalité"}, + "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, + "label": {"en": "Label", "de": "Bezeichnung", "fr": "Libellé"}, + "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, + }, +) diff --git a/modules/datamodels/datamodelInvitation.py b/modules/datamodels/datamodelInvitation.py new file mode 100644 index 00000000..a35dfb09 --- /dev/null +++ b/modules/datamodels/datamodelInvitation.py @@ -0,0 +1,120 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Invitation model for self-service onboarding. +Token-basierte Einladungen für neue User zu Mandanten/Features. +""" + +import uuid +import secrets +from typing import Optional, List +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp + + +class Invitation(BaseModel): + """ + Einladungs-Token für neue User. + Ermöglicht Self-Service Onboarding zu Mandanten und Feature-Instanzen. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the invitation", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + token: str = Field( + default_factory=lambda: secrets.token_urlsafe(32), + description="Secure invitation token", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Ziel der Einladung + mandateId: str = Field( + description="FK → Mandate.id - Target mandate for the invitation", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Optional FK → FeatureInstance.id - Direct access to specific feature", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + roleIds: List[str] = Field( + default_factory=list, + description="List of Role IDs to assign to the invited user", + json_schema_extra={"frontend_type": "multiselect", "frontend_readonly": False, "frontend_required": True} + ) + + # Einladungs-Details + email: Optional[str] = Field( + default=None, + description="Target email address (optional, for tracking)", + json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": False} + ) + createdBy: str = Field( + description="User ID of the person who created the invitation", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + createdAt: float = Field( + default_factory=getUtcTimestamp, + description="When the invitation was created (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + expiresAt: float = Field( + description="When the invitation expires (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": True} + ) + + # Status + usedBy: Optional[str] = Field( + default=None, + description="User ID of the person who used the invitation", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + usedAt: Optional[float] = Field( + default=None, + description="When the invitation was used (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + revokedAt: Optional[float] = Field( + default=None, + description="When the invitation was revoked (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + + # Einschränkungen + maxUses: int = Field( + default=1, + ge=1, + le=100, + description="Maximum number of times this invitation can be used", + json_schema_extra={"frontend_type": "number", "frontend_readonly": False, "frontend_required": False} + ) + currentUses: int = Field( + default=0, + ge=0, + description="Current number of times this invitation has been used", + json_schema_extra={"frontend_type": "number", "frontend_readonly": True, "frontend_required": False} + ) + + +registerModelLabels( + "Invitation", + {"en": "Invitation", "de": "Einladung", "fr": "Invitation"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "token": {"en": "Token", "de": "Token", "fr": "Jeton"}, + "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance"}, + "roleIds": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, + "email": {"en": "Email", "de": "E-Mail", "fr": "Email"}, + "createdBy": {"en": "Created By", "de": "Erstellt von", "fr": "Créé par"}, + "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, + "expiresAt": {"en": "Expires At", "de": "Gültig bis", "fr": "Expire le"}, + "usedBy": {"en": "Used By", "de": "Verwendet von", "fr": "Utilisé par"}, + "usedAt": {"en": "Used At", "de": "Verwendet am", "fr": "Utilisé le"}, + "revokedAt": {"en": "Revoked At", "de": "Widerrufen am", "fr": "Révoqué le"}, + "maxUses": {"en": "Max Uses", "de": "Max. Verwendungen", "fr": "Utilisations max"}, + "currentUses": {"en": "Current Uses", "de": "Aktuelle Verwendungen", "fr": "Utilisations actuelles"}, + }, +) diff --git a/modules/datamodels/datamodelMembership.py b/modules/datamodels/datamodelMembership.py new file mode 100644 index 00000000..e2cdb0b6 --- /dev/null +++ b/modules/datamodels/datamodelMembership.py @@ -0,0 +1,150 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Membership models: UserMandate, FeatureAccess, and Junction Tables. + +Diese Models definieren die m:n Beziehungen zwischen User, Mandate und FeatureInstance. +Rollen werden über Junction Tables verknüpft für saubere CASCADE DELETE. +""" + +import uuid +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels + + +class UserMandate(BaseModel): + """ + User-Mitgliedschaft in einem Mandanten. + Kein User gehört direkt zu einem Mandanten - Zugehörigkeit wird über dieses Model gesteuert. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the user-mandate membership", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + userId: str = Field( + description="FK → User.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + mandateId: str = Field( + description="FK → Mandate.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + enabled: bool = Field( + default=True, + description="Whether this membership is enabled", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) + # Rollen werden via Junction Table UserMandateRole verknüpft + + +registerModelLabels( + "UserMandate", + {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, + "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, + "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, + }, +) + + +class FeatureAccess(BaseModel): + """ + User-Zugriff auf eine Feature-Instanz. + Definiert welche User auf welche Feature-Instanzen zugreifen können. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the feature access", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + userId: str = Field( + description="FK → User.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + featureInstanceId: str = Field( + description="FK → FeatureInstance.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + enabled: bool = Field( + default=True, + description="Whether this feature access is enabled", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) + # Rollen werden via Junction Table FeatureAccessRole verknüpft + + +registerModelLabels( + "FeatureAccess", + {"en": "Feature Access", "de": "Feature-Zugang", "fr": "Accès fonctionnalité"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, + "featureInstanceId": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance"}, + "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, + }, +) + + +class UserMandateRole(BaseModel): + """ + Junction Table: UserMandate zu Role. + Ermöglicht CASCADE DELETE auf Datenbankebene. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the junction record", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + userMandateId: str = Field( + description="FK → UserMandate.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + roleId: str = Field( + description="FK → Role.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + +registerModelLabels( + "UserMandateRole", + {"en": "User Mandate Role", "de": "Benutzer-Mandant-Rolle", "fr": "Rôle mandat utilisateur"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userMandateId": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, + "roleId": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, + }, +) + + +class FeatureAccessRole(BaseModel): + """ + Junction Table: FeatureAccess zu Role. + Ermöglicht CASCADE DELETE auf Datenbankebene. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the junction record", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureAccessId: str = Field( + description="FK → FeatureAccess.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + roleId: str = Field( + description="FK → Role.id (CASCADE DELETE)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + +registerModelLabels( + "FeatureAccessRole", + {"en": "Feature Access Role", "de": "Feature-Zugang-Rolle", "fr": "Rôle accès fonctionnalité"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "featureAccessId": {"en": "Feature Access", "de": "Feature-Zugang", "fr": "Accès fonctionnalité"}, + "roleId": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, + }, +) diff --git a/modules/datamodels/datamodelRbac.py b/modules/datamodels/datamodelRbac.py index c9be666f..666470c8 100644 --- a/modules/datamodels/datamodelRbac.py +++ b/modules/datamodels/datamodelRbac.py @@ -1,9 +1,16 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""RBAC models: AccessRule, AccessRuleContext, Role.""" +""" +RBAC models: AccessRule, AccessRuleContext, Role. + +Multi-Tenant Design: +- Role hat einen unveränderlichen Kontext (mandateId, featureInstanceId, featureCode) +- AccessRule referenziert Role via roleId (FK), nicht via roleLabel +- Kontext-Felder sind IMMUTABLE nach Erstellung +""" import uuid -from typing import Optional, Dict +from typing import Optional from enum import Enum from pydantic import BaseModel, Field from modules.shared.attributeUtils import registerModelLabels @@ -19,7 +26,17 @@ class AccessRuleContext(str, Enum): class Role(BaseModel): - """Data model for RBAC roles""" + """ + Data model for RBAC roles. + + Kernkonzept: Eine Rolle existiert immer in einem spezifischen Kontext. + Der Kontext ist IMMUTABLE nach Erstellung. + + Kontext-Typen: + - mandateId=None, featureInstanceId=None → GLOBAL (Template-Rolle) + - mandateId=X, featureInstanceId=None → MANDATE-Rolle + - mandateId=X, featureInstanceId=Y → INSTANCE-Rolle + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the role", @@ -33,106 +50,163 @@ class Role(BaseModel): description="Role description in multiple languages", json_schema_extra={"frontend_type": "multilingual", "frontend_readonly": False, "frontend_required": True} ) + + # KONTEXT - IMMUTABLE nach Create (nur Create/Delete, kein Update!) + mandateId: Optional[str] = Field( + default=None, + description="FK → Mandate.id (CASCADE DELETE). Null = Global/Template role.", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureInstanceId: Optional[str] = Field( + default=None, + description="FK → FeatureInstance.id (CASCADE DELETE). Null = Mandate-level or Global role.", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureCode: Optional[str] = Field( + default=None, + description="Feature code (z.B. 'trustee') - für Template-Rollen", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + isSystemRole: bool = Field( - False, + default=False, description="Whether this is a system role that cannot be deleted", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": False} ) + registerModelLabels( "Role", - {"en": "Role", "fr": "Rôle"}, + {"en": "Role", "de": "Rolle", "fr": "Rôle"}, { - "id": {"en": "ID", "fr": "ID"}, - "roleLabel": {"en": "Role Label", "fr": "Label du rôle"}, - "description": {"en": "Description", "fr": "Description"}, - "isSystemRole": {"en": "System Role", "fr": "Rôle système"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "roleLabel": {"en": "Role Label", "de": "Rollen-Label", "fr": "Label du rôle"}, + "description": {"en": "Description", "de": "Beschreibung", "fr": "Description"}, + "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance"}, + "featureCode": {"en": "Feature Code", "de": "Feature-Code", "fr": "Code fonctionnalité"}, + "isSystemRole": {"en": "System Role", "de": "System-Rolle", "fr": "Rôle système"}, }, ) class AccessRule(BaseModel): - """Data model for access control rules""" + """ + Data model for access control rules. + + WICHTIG: roleId referenziert die Role via FK (nicht mehr roleLabel!) + Die Felder 'context' und 'roleId' sind IMMUTABLE nach Erstellung. + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the access rule", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) - roleLabel: str = Field( - description="Role label this rule applies to", - json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_options": "user.role"} + roleId: str = Field( + description="FK → Role.id (CASCADE DELETE!)", + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": True} ) context: AccessRuleContext = Field( - description="Context type: DATA (database), UI (interface), RESOURCE (system resources)", - json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_options": [ - {"value": "DATA", "label": {"en": "Data", "fr": "Données"}}, - {"value": "UI", "label": {"en": "UI", "fr": "Interface"}}, - {"value": "RESOURCE", "label": {"en": "Resource", "fr": "Ressource"}} + description="Context type: DATA (database), UI (interface), RESOURCE (system resources). IMMUTABLE!", + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": True, "frontend_options": [ + {"value": "DATA", "label": {"en": "Data", "de": "Daten", "fr": "Données"}}, + {"value": "UI", "label": {"en": "UI", "de": "Oberfläche", "fr": "Interface"}}, + {"value": "RESOURCE", "label": {"en": "Resource", "de": "Ressource", "fr": "Ressource"}} ]} ) item: Optional[str] = Field( - None, + default=None, description="Item identifier (null = all items in context). Format: DATA: '' or '
.', UI: cascading string (e.g., 'playground.voice.settings'), RESOURCE: cascading string (e.g., 'ai.model.anthropic')", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False} ) view: bool = Field( - False, + default=False, description="View permission: if true, item is visible/enabled. Only objects with view=true are shown.", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": True} ) read: Optional[AccessLevel] = Field( - None, + default=None, description="Read permission level (only for DATA context)", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ - {"value": "a", "label": {"en": "All Records", "fr": "Tous les enregistrements"}}, - {"value": "m", "label": {"en": "My Records", "fr": "Mes enregistrements"}}, - {"value": "g", "label": {"en": "Group Records", "fr": "Enregistrements du groupe"}}, - {"value": "n", "label": {"en": "No Access", "fr": "Aucun accès"}} + {"value": "a", "label": {"en": "All Records", "de": "Alle Datensätze", "fr": "Tous les enregistrements"}}, + {"value": "m", "label": {"en": "My Records", "de": "Meine Datensätze", "fr": "Mes enregistrements"}}, + {"value": "g", "label": {"en": "Group Records", "de": "Gruppen-Datensätze", "fr": "Enregistrements du groupe"}}, + {"value": "n", "label": {"en": "No Access", "de": "Kein Zugriff", "fr": "Aucun accès"}} ]} ) create: Optional[AccessLevel] = Field( - None, + default=None, description="Create permission level (only for DATA context)", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ - {"value": "a", "label": {"en": "All Records", "fr": "Tous les enregistrements"}}, - {"value": "m", "label": {"en": "My Records", "fr": "Mes enregistrements"}}, - {"value": "g", "label": {"en": "Group Records", "fr": "Enregistrements du groupe"}}, - {"value": "n", "label": {"en": "No Access", "fr": "Aucun accès"}} + {"value": "a", "label": {"en": "All Records", "de": "Alle Datensätze", "fr": "Tous les enregistrements"}}, + {"value": "m", "label": {"en": "My Records", "de": "Meine Datensätze", "fr": "Mes enregistrements"}}, + {"value": "g", "label": {"en": "Group Records", "de": "Gruppen-Datensätze", "fr": "Enregistrements du groupe"}}, + {"value": "n", "label": {"en": "No Access", "de": "Kein Zugriff", "fr": "Aucun accès"}} ]} ) update: Optional[AccessLevel] = Field( - None, + default=None, description="Update permission level (only for DATA context)", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ - {"value": "a", "label": {"en": "All Records", "fr": "Tous les enregistrements"}}, - {"value": "m", "label": {"en": "My Records", "fr": "Mes enregistrements"}}, - {"value": "g", "label": {"en": "Group Records", "fr": "Enregistrements du groupe"}}, - {"value": "n", "label": {"en": "No Access", "fr": "Aucun accès"}} + {"value": "a", "label": {"en": "All Records", "de": "Alle Datensätze", "fr": "Tous les enregistrements"}}, + {"value": "m", "label": {"en": "My Records", "de": "Meine Datensätze", "fr": "Mes enregistrements"}}, + {"value": "g", "label": {"en": "Group Records", "de": "Gruppen-Datensätze", "fr": "Enregistrements du groupe"}}, + {"value": "n", "label": {"en": "No Access", "de": "Kein Zugriff", "fr": "Aucun accès"}} ]} ) delete: Optional[AccessLevel] = Field( - None, + default=None, description="Delete permission level (only for DATA context)", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ - {"value": "a", "label": {"en": "All Records", "fr": "Tous les enregistrements"}}, - {"value": "m", "label": {"en": "My Records", "fr": "Mes enregistrements"}}, - {"value": "g", "label": {"en": "Group Records", "fr": "Enregistrements du groupe"}}, - {"value": "n", "label": {"en": "No Access", "fr": "Aucun accès"}} + {"value": "a", "label": {"en": "All Records", "de": "Alle Datensätze", "fr": "Tous les enregistrements"}}, + {"value": "m", "label": {"en": "My Records", "de": "Meine Datensätze", "fr": "Mes enregistrements"}}, + {"value": "g", "label": {"en": "Group Records", "de": "Gruppen-Datensätze", "fr": "Enregistrements du groupe"}}, + {"value": "n", "label": {"en": "No Access", "de": "Kein Zugriff", "fr": "Aucun accès"}} ]} ) + registerModelLabels( "AccessRule", - {"en": "Access Rule", "fr": "Règle d'accès"}, + {"en": "Access Rule", "de": "Zugriffsregel", "fr": "Règle d'accès"}, { - "id": {"en": "ID", "fr": "ID"}, - "roleLabel": {"en": "Role Label", "fr": "Label du rôle"}, - "context": {"en": "Context", "fr": "Contexte"}, - "item": {"en": "Item", "fr": "Élément"}, - "view": {"en": "View", "fr": "Vue"}, - "read": {"en": "Read", "fr": "Lecture"}, - "create": {"en": "Create", "fr": "Créer"}, - "update": {"en": "Update", "fr": "Mettre à jour"}, - "delete": {"en": "Delete", "fr": "Supprimer"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "roleId": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, + "context": {"en": "Context", "de": "Kontext", "fr": "Contexte"}, + "item": {"en": "Item", "de": "Element", "fr": "Élément"}, + "view": {"en": "View", "de": "Anzeigen", "fr": "Vue"}, + "read": {"en": "Read", "de": "Lesen", "fr": "Lecture"}, + "create": {"en": "Create", "de": "Erstellen", "fr": "Créer"}, + "update": {"en": "Update", "de": "Aktualisieren", "fr": "Mettre à jour"}, + "delete": {"en": "Delete", "de": "Löschen", "fr": "Supprimer"}, }, ) + + +# IMMUTABLE Fields Definition - für Enforcement auf Application-Level +IMMUTABLE_FIELDS = { + "Role": ["mandateId", "featureInstanceId", "featureCode"], + "AccessRule": ["context", "roleId"] +} + + +def validateUpdateNotImmutable(model: str, updateData: dict): + """ + Blockiert Updates auf immutable Felder. + Wirft ValueError wenn versucht wird, Kontext-Felder zu ändern. + + Args: + model: Model name (z.B. "Role", "AccessRule") + updateData: Dictionary mit Update-Daten + + Raises: + ValueError: Wenn immutable Felder im Update enthalten sind + """ + forbidden = IMMUTABLE_FIELDS.get(model, []) + violations = [f for f in forbidden if f in updateData] + + if violations: + raise ValueError( + f"Cannot update immutable fields on {model}: {violations}. " + f"Delete and recreate instead." + ) diff --git a/modules/datamodels/datamodelSecurity.py b/modules/datamodels/datamodelSecurity.py index eac51430..9fed9fa4 100644 --- a/modules/datamodels/datamodelSecurity.py +++ b/modules/datamodels/datamodelSecurity.py @@ -1,6 +1,13 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""Security models: Token and AuthEvent.""" +""" +Security models: Token and AuthEvent. + +Multi-Tenant Design: +- Token ist NICHT an einen Mandanten gebunden +- User arbeitet parallel in mehreren Mandanten (z.B. mehrere Browser-Tabs) +- Mandant-Kontext wird per Request-Header (X-Mandate-Id) bestimmt +""" from typing import Optional from pydantic import BaseModel, Field, ConfigDict @@ -17,6 +24,14 @@ class TokenStatus(str, Enum): class Token(BaseModel): + """ + Authentication Token model. + + Multi-Tenant Design: + - Token ist User-gebunden, NICHT Mandant-gebunden + - Ermöglicht parallele Arbeit in mehreren Mandanten + - Mandant-Kontext wird per Request-Header bestimmt + """ id: Optional[str] = None userId: str authority: AuthAuthority @@ -45,37 +60,36 @@ class Token(BaseModel): sessionId: Optional[str] = Field( None, description="Logical session grouping for logout revocation" ) - mandateId: Optional[str] = Field( - None, description="Mandate ID for tenant scoping of the token" - ) + # ENTFERNT: mandateId - Token ist nicht mehr Mandant-spezifisch + # Mandant-Kontext wird per Request-Header (X-Mandate-Id) bestimmt model_config = ConfigDict(use_enum_values=True) registerModelLabels( "Token", - {"en": "Token", "fr": "Jeton"}, + {"en": "Token", "de": "Token", "fr": "Jeton"}, { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "authority": {"en": "Authority", "fr": "Autorité"}, - "connectionId": {"en": "Connection ID", "fr": "ID de connexion"}, - "tokenAccess": {"en": "Access Token", "fr": "Jeton d'accès"}, - "tokenType": {"en": "Token Type", "fr": "Type de jeton"}, - "expiresAt": {"en": "Expires At", "fr": "Expire le"}, - "tokenRefresh": {"en": "Refresh Token", "fr": "Jeton de rafraîchissement"}, - "createdAt": {"en": "Created At", "fr": "Créé le"}, - "status": {"en": "Status", "fr": "Statut"}, - "revokedAt": {"en": "Revoked At", "fr": "Révoqué le"}, - "revokedBy": {"en": "Revoked By", "fr": "Révoqué par"}, - "reason": {"en": "Reason", "fr": "Raison"}, - "sessionId": {"en": "Session ID", "fr": "ID de session"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "authority": {"en": "Authority", "de": "Autorität", "fr": "Autorité"}, + "connectionId": {"en": "Connection ID", "de": "Verbindungs-ID", "fr": "ID de connexion"}, + "tokenAccess": {"en": "Access Token", "de": "Zugriffstoken", "fr": "Jeton d'accès"}, + "tokenType": {"en": "Token Type", "de": "Token-Typ", "fr": "Type de jeton"}, + "expiresAt": {"en": "Expires At", "de": "Läuft ab am", "fr": "Expire le"}, + "tokenRefresh": {"en": "Refresh Token", "de": "Refresh-Token", "fr": "Jeton de rafraîchissement"}, + "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "revokedAt": {"en": "Revoked At", "de": "Widerrufen am", "fr": "Révoqué le"}, + "revokedBy": {"en": "Revoked By", "de": "Widerrufen von", "fr": "Révoqué par"}, + "reason": {"en": "Reason", "de": "Grund", "fr": "Raison"}, + "sessionId": {"en": "Session ID", "de": "Sitzungs-ID", "fr": "ID de session"}, }, ) class AuthEvent(BaseModel): + """Authentication event for audit logging.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the auth event", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user this event belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) eventType: str = Field(description="Type of authentication event (e.g., 'login', 'logout', 'token_refresh')", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) @@ -88,15 +102,15 @@ class AuthEvent(BaseModel): registerModelLabels( "AuthEvent", - {"en": "Authentication Event", "fr": "Événement d'authentification"}, + {"en": "Authentication Event", "de": "Authentifizierungsereignis", "fr": "Événement d'authentification"}, { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "eventType": {"en": "Event Type", "fr": "Type d'événement"}, - "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, - "ipAddress": {"en": "IP Address", "fr": "Adresse IP"}, - "userAgent": {"en": "User Agent", "fr": "Agent utilisateur"}, - "success": {"en": "Success", "fr": "Succès"}, - "details": {"en": "Details", "fr": "Détails"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "eventType": {"en": "Event Type", "de": "Ereignistyp", "fr": "Type d'événement"}, + "timestamp": {"en": "Timestamp", "de": "Zeitstempel", "fr": "Horodatage"}, + "ipAddress": {"en": "IP Address", "de": "IP-Adresse", "fr": "Adresse IP"}, + "userAgent": {"en": "User Agent", "de": "User-Agent", "fr": "Agent utilisateur"}, + "success": {"en": "Success", "de": "Erfolgreich", "fr": "Succès"}, + "details": {"en": "Details", "de": "Details", "fr": "Détails"}, }, ) diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index adea38b0..f1c5da33 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -1,11 +1,18 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""UAM models: User, Mandate, UserConnection.""" +""" +UAM models: User, Mandate, UserConnection. + +Multi-Tenant Design: +- User gehört NICHT direkt zu einem Mandanten +- Zugehörigkeit wird über UserMandate gesteuert (siehe datamodelMembership.py) +- isSysAdmin ist globales Admin-Flag für System-Zugriff (KEIN Daten-Zugriff!) +""" import uuid from typing import Optional, List from enum import Enum -from pydantic import BaseModel, Field, EmailStr +from pydantic import BaseModel, Field, EmailStr, field_validator from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp @@ -51,7 +58,12 @@ class UserPermissions(BaseModel): description="Delete permission level" ) + class Mandate(BaseModel): + """ + Mandate (Mandant/Tenant) model. + Ein Mandant ist ein isolierter Bereich für Daten und Berechtigungen. + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the mandate", @@ -61,37 +73,24 @@ class Mandate(BaseModel): description="Name of the mandate", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} ) - language: str = Field( - default="en", - description="Default language of the mandate", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": [ - {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, - {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, - {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, - {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, - ] - } - ) enabled: bool = Field( default=True, description="Indicates whether the mandate is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} ) + + registerModelLabels( "Mandate", - {"en": "Mandate", "fr": "Mandat"}, + {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, { - "id": {"en": "ID", "fr": "ID"}, - "name": {"en": "Name", "fr": "Nom"}, - "language": {"en": "Language", "fr": "Langue"}, - "enabled": {"en": "Enabled", "fr": "Activé"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "name": {"en": "Name", "de": "Name", "fr": "Nom"}, + "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, }, ) + class UserConnection(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the connection", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user this connection belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) @@ -109,70 +108,123 @@ class UserConnection(BaseModel): {"value": "none", "label": {"en": "None", "fr": "Aucun"}}, ]}) tokenExpiresAt: Optional[float] = Field(None, description="When the current token expires (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + + registerModelLabels( "UserConnection", - {"en": "User Connection", "fr": "Connexion utilisateur"}, + {"en": "User Connection", "de": "Benutzerverbindung", "fr": "Connexion utilisateur"}, { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "authority": {"en": "Authority", "fr": "Autorité"}, - "externalId": {"en": "External ID", "fr": "ID externe"}, - "externalUsername": {"en": "External Username", "fr": "Nom d'utilisateur externe"}, - "externalEmail": {"en": "External Email", "fr": "Email externe"}, - "status": {"en": "Status", "fr": "Statut"}, - "connectedAt": {"en": "Connected At", "fr": "Connecté le"}, - "lastChecked": {"en": "Last Checked", "fr": "Dernière vérification"}, - "expiresAt": {"en": "Expires At", "fr": "Expire le"}, - "tokenStatus": {"en": "Connection Status", "fr": "Statut de connexion"}, - "tokenExpiresAt": {"en": "Expires At", "fr": "Expire le"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "authority": {"en": "Authority", "de": "Autorität", "fr": "Autorité"}, + "externalId": {"en": "External ID", "de": "Externe ID", "fr": "ID externe"}, + "externalUsername": {"en": "External Username", "de": "Externer Benutzername", "fr": "Nom d'utilisateur externe"}, + "externalEmail": {"en": "External Email", "de": "Externe E-Mail", "fr": "Email externe"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "connectedAt": {"en": "Connected At", "de": "Verbunden am", "fr": "Connecté le"}, + "lastChecked": {"en": "Last Checked", "de": "Zuletzt geprüft", "fr": "Dernière vérification"}, + "expiresAt": {"en": "Expires At", "de": "Läuft ab am", "fr": "Expire le"}, + "tokenStatus": {"en": "Connection Status", "de": "Verbindungsstatus", "fr": "Statut de connexion"}, + "tokenExpiresAt": {"en": "Expires At", "de": "Läuft ab am", "fr": "Expire le"}, }, ) + class User(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the user", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - username: str = Field(description="Username for login", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) - email: Optional[EmailStr] = Field(None, description="Email address of the user", json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": True}) - fullName: Optional[str] = Field(None, description="Full name of the user", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) - language: str = Field(default="en", description="Preferred language of the user", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_options": [ - {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, - {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, - {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, - {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, - ]}) - enabled: bool = Field(default=True, description="Indicates whether the user is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) - roleLabels: List[str] = Field( - default_factory=list, - description="List of role labels assigned to this user. All roles are opening roles (union) - if one role enables something, it is enabled.", - json_schema_extra={"frontend_type": "multiselect", "frontend_readonly": False, "frontend_required": True, "frontend_options": "user.role"} + """ + User model. + + Multi-Tenant Design: + - User gehört NICHT direkt zu einem Mandanten + - Zugehörigkeit wird über UserMandate gesteuert (siehe datamodelMembership.py) + - Rollen werden über UserMandateRole gesteuert + - isSysAdmin = System-Zugriff, KEIN Daten-Zugriff + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the user", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) - authenticationAuthority: AuthAuthority = Field(default=AuthAuthority.LOCAL, description="Primary authentication authority", json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "auth.authority"}) - mandateId: Optional[str] = Field(None, description="ID of the mandate this user belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + username: str = Field( + description="Username for login", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} + ) + email: Optional[EmailStr] = Field( + default=None, + description="Email address of the user", + json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": True} + ) + fullName: Optional[str] = Field( + default=None, + description="Full name of the user", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False} + ) + language: str = Field( + default="en", + description="Preferred language of the user", + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_options": [ + {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, + {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, + {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, + {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, + ]} + ) + enabled: bool = Field( + default=True, + description="Indicates whether the user is enabled", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) + + isSysAdmin: bool = Field( + default=False, + description="Global SysAdmin flag. SysAdmin = System-Zugriff, KEIN Daten-Zugriff!", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) + + @field_validator('isSysAdmin', mode='before') + @classmethod + def _coerceIsSysAdmin(cls, v): + """Konvertiert None zu False (für bestehende DB-Einträge ohne isSysAdmin Feld).""" + if v is None: + return False + return v + + authenticationAuthority: AuthAuthority = Field( + default=AuthAuthority.LOCAL, + description="Primary authentication authority", + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "auth.authority"} + ) + + registerModelLabels( "User", - {"en": "User", "fr": "Utilisateur"}, + {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, { - "id": {"en": "ID", "fr": "ID"}, - "username": {"en": "Username", "fr": "Nom d'utilisateur"}, - "email": {"en": "Email", "fr": "Email"}, - "fullName": {"en": "Full Name", "fr": "Nom complet"}, - "language": {"en": "Language", "fr": "Langue"}, - "enabled": {"en": "Enabled", "fr": "Activé"}, - "roleLabels": {"en": "Role Labels", "fr": "Labels de rôle"}, - "authenticationAuthority": {"en": "Auth Authority", "fr": "Autorité d'authentification"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "username": {"en": "Username", "de": "Benutzername", "fr": "Nom d'utilisateur"}, + "email": {"en": "Email", "de": "E-Mail", "fr": "Email"}, + "fullName": {"en": "Full Name", "de": "Vollständiger Name", "fr": "Nom complet"}, + "language": {"en": "Language", "de": "Sprache", "fr": "Langue"}, + "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, + "isSysAdmin": {"en": "System Admin", "de": "System-Admin", "fr": "Admin système"}, + "authenticationAuthority": {"en": "Auth Authority", "de": "Authentifizierung", "fr": "Autorité d'authentification"}, }, ) + class UserInDB(User): + """User model with password hash for database storage.""" hashedPassword: Optional[str] = Field(None, description="Hash of the user password") resetToken: Optional[str] = Field(None, description="Password reset token (UUID)") resetTokenExpires: Optional[float] = Field(None, description="Reset token expiration (UTC timestamp in seconds)") + + registerModelLabels( "UserInDB", - {"en": "User Access", "fr": "Accès de l'utilisateur"}, + {"en": "User Access", "de": "Benutzerzugang", "fr": "Accès de l'utilisateur"}, { - "hashedPassword": {"en": "Password hash", "fr": "Hachage de mot de passe"}, - "resetToken": {"en": "Reset Token", "fr": "Jeton de réinitialisation"}, - "resetTokenExpires": {"en": "Reset Token Expires", "fr": "Expiration du jeton"}, + "hashedPassword": {"en": "Password hash", "de": "Passwort-Hash", "fr": "Hachage de mot de passe"}, + "resetToken": {"en": "Reset Token", "de": "Reset-Token", "fr": "Jeton de réinitialisation"}, + "resetTokenExpires": {"en": "Reset Token Expires", "de": "Token läuft ab", "fr": "Expiration du jeton"}, }, ) diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index eaaf8b43..43503339 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -62,6 +62,7 @@ def _extractJsonFromResponse(content: str) -> Optional[dict]: async def chatProcess( currentUser: User, + mandateId: str, userInput: UserInputRequest, workflowId: Optional[str] = None ) -> ChatWorkflow: @@ -76,6 +77,7 @@ async def chatProcess( Args: currentUser: Current user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) userInput: User input request workflowId: Optional workflow ID to continue existing conversation @@ -83,8 +85,8 @@ async def chatProcess( ChatWorkflow instance """ try: - # Get services - services = getServices(currentUser, None) + # Get services with mandate context + services = getServices(currentUser, None, mandateId=mandateId) interfaceDbChat = services.interfaceDbChat # Get event manager and create queue if needed @@ -120,7 +122,7 @@ async def chatProcess( # Create new workflow workflowData = { "id": str(uuid.uuid4()), - "mandateId": currentUser.mandateId, + "mandateId": mandateId, "status": "running", "name": conversation_name, "currentRound": 1, @@ -687,12 +689,13 @@ async def _convert_file_ids_to_document_references( # Search database if not found in messages if not document_id: try: - from modules.shared.databaseUtils import getRecordsetWithRBAC + from modules.interfaces.interfaceRbac import getRecordsetWithRBAC documents = getRecordsetWithRBAC( services.interfaceDbChat.db, ChatDocument, - services.currentUser, - recordFilter={"fileId": file_id} + services.user, + recordFilter={"fileId": file_id}, + mandateId=services.mandateId ) if documents: workflow_message_ids = {msg.id for msg in workflow.messages} if workflow.messages else set() diff --git a/modules/features/dynamicOptions/mainDynamicOptions.py b/modules/features/dynamicOptions/mainDynamicOptions.py index b75e906b..e8c9a4ff 100644 --- a/modules/features/dynamicOptions/mainDynamicOptions.py +++ b/modules/features/dynamicOptions/mainDynamicOptions.py @@ -126,7 +126,7 @@ def getOptions(optionsName: str, services, currentUser: Optional[User] = None) - return [] try: - users = services.interfaceDbApp.getUsersByMandate(currentUser.mandateId) + users = services.interfaceDbApp.getUsersByMandate(services.mandateId) # Handle both list and PaginatedResult if hasattr(users, 'items'): diff --git a/modules/features/neutralizePlayground/mainNeutralizePlayground.py b/modules/features/neutralizePlayground/mainNeutralizePlayground.py index 80e0e03b..d10dc8ec 100644 --- a/modules/features/neutralizePlayground/mainNeutralizePlayground.py +++ b/modules/features/neutralizePlayground/mainNeutralizePlayground.py @@ -15,9 +15,10 @@ logger = logging.getLogger(__name__) class NeutralizationPlayground: """Feature/UI wrapper around NeutralizationService for playground & routes.""" - def __init__(self, currentUser: User): + def __init__(self, currentUser: User, mandateId: str): self.currentUser = currentUser - self.services = getServices(currentUser, None) + self.mandateId = mandateId + self.services = getServices(currentUser, None, mandateId=mandateId) def processText(self, text: str) -> Dict[str, Any]: return self.services.neutralization.processText(text) @@ -81,7 +82,7 @@ class NeutralizationPlayground: 'total_attributes': len(allAttributes), 'unique_files': len(uniqueFiles), 'pattern_counts': patternCounts, - 'mandate_id': self.currentUser.mandateId if self.currentUser else None, + 'mandate_id': self.mandateId, } except Exception as e: logger.error(f"Error getting stats: {str(e)}") diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 149fd8d8..37c4a3cd 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -346,6 +346,7 @@ async def fetch_parcel_polygon_from_swisstopo( async def executeDirectQuery( currentUser: User, + mandateId: str, queryText: str, parameters: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: @@ -354,6 +355,7 @@ async def executeDirectQuery( Args: currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) queryText: SQL query text parameters: Optional parameters for parameterized queries @@ -364,16 +366,15 @@ async def executeDirectQuery( - No session or query history is saved - Query is executed directly and result is returned - For production, validate and sanitize queries before execution - - TODO: Implement actual database query execution via interface """ try: - logger.info(f"Executing direct query for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Executing direct query for user {currentUser.id} (mandate: {mandateId})") logger.debug(f"Query text: {queryText}") if parameters: logger.debug(f"Query parameters: {parameters}") # Execute query via Real Estate interface (stateless) - realEstateInterface = getRealEstateInterface(currentUser) + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) result = realEstateInterface.executeQuery(queryText, parameters) logger.info( @@ -529,6 +530,7 @@ def _formatEntitySummary(entity_type: str, items: List[Dict[str, Any]], filters: async def processNaturalLanguageCommand( currentUser: User, + mandateId: str, userInput: str, ) -> Dict[str, Any]: """ @@ -539,6 +541,7 @@ async def processNaturalLanguageCommand( Args: currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) userInput: Natural language command from user Returns: @@ -552,11 +555,11 @@ async def processNaturalLanguageCommand( - "SELECT * FROM Projekt WHERE plz = '8000'" """ try: - logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {mandateId})") logger.debug(f"User input: {userInput}") # Initialize services for AI access - services = getServices(currentUser, workflow=None) + services = getServices(currentUser, workflow=None, mandateId=mandateId) aiService = services.ai # Step 1: Analyze user intent with AI @@ -567,6 +570,7 @@ async def processNaturalLanguageCommand( # Step 2: Execute CRUD operation based on intent result = await executeIntentBasedOperation( currentUser=currentUser, + mandateId=mandateId, intent=intentAnalysis["intent"], entity=intentAnalysis.get("entity"), parameters=intentAnalysis.get("parameters", {}), @@ -839,6 +843,7 @@ IMPORTANT EXTRACTION RULES: async def executeIntentBasedOperation( currentUser: User, + mandateId: str, intent: str, entity: Optional[str], parameters: Dict[str, Any], @@ -848,6 +853,7 @@ async def executeIntentBasedOperation( Args: currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) entity: Entity type from AI analysis parameters: Extracted parameters from AI analysis @@ -856,8 +862,8 @@ async def executeIntentBasedOperation( Operation result Note: - - TODO: Implement actual interface calls once datamodels are ready - - Currently returns test responses showing what would be executed + - Supports CREATE, READ, UPDATE, DELETE, QUERY intents + - Entity types: Projekt, Parzelle, Gemeinde, Kanton, Land, Dokument """ try: logger.info(f"Executing intent-based operation: intent={intent}, entity={entity}") @@ -872,6 +878,7 @@ async def executeIntentBasedOperation( result = await executeDirectQuery( currentUser=currentUser, + mandateId=mandateId, queryText=queryText, parameters=parameters.get("queryParameters"), ) @@ -879,12 +886,12 @@ async def executeIntentBasedOperation( elif intent == "CREATE": # Create new entity - realEstateInterface = getRealEstateInterface(currentUser) + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) if entity == "Projekt": # Create Projekt from parameters projekt = Projekt( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=parameters.get("label", ""), statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, ) @@ -902,7 +909,7 @@ async def executeIntentBasedOperation( # Build parzelle data with all extracted parameters parzelle_data = { - "mandateId": currentUser.mandateId, + "mandateId": mandateId, "label": parameters.get("label", ""), } @@ -985,7 +992,7 @@ async def executeIntentBasedOperation( # Create Gemeinde from parameters from modules.datamodels.datamodelRealEstate import Gemeinde gemeinde = Gemeinde( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=parameters.get("label", ""), id_kanton=parameters.get("id_kanton"), plz=parameters.get("plz"), @@ -1000,7 +1007,7 @@ async def executeIntentBasedOperation( # Create Kanton from parameters from modules.datamodels.datamodelRealEstate import Kanton kanton = Kanton( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=parameters.get("label", ""), id_land=parameters.get("id_land"), abk=parameters.get("abk"), @@ -1015,7 +1022,7 @@ async def executeIntentBasedOperation( # Create Land from parameters from modules.datamodels.datamodelRealEstate import Land land = Land( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=parameters.get("label", ""), abk=parameters.get("abk"), ) @@ -1029,7 +1036,7 @@ async def executeIntentBasedOperation( # Create Dokument from parameters from modules.datamodels.datamodelRealEstate import Dokument dokument = Dokument( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=parameters.get("label", ""), dokumentReferenz=parameters.get("dokumentReferenz", ""), versionsbezeichnung=parameters.get("versionsbezeichnung"), @@ -1474,6 +1481,7 @@ async def executeIntentBasedOperation( async def create_project_with_parcel_data( currentUser: User, + mandateId: str, projekt_label: str, parzellen_data: List[Dict[str, Any]], status_prozess: Optional[str] = None, @@ -1483,6 +1491,7 @@ async def create_project_with_parcel_data( Args: currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) projekt_label: Label for the Projekt parzellen_data: List of dictionaries containing parcel information from request status_prozess: Optional project status (defaults to "Eingang") @@ -1496,8 +1505,8 @@ async def create_project_with_parcel_data( try: logger.info(f"Creating project '{projekt_label}' with {len(parzellen_data)} parcel(s) for user {currentUser.id}") - # Get interface - realEstateInterface = getRealEstateInterface(currentUser) + # Get interface with mandate context + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) # Validate required fields if not projekt_label: @@ -1587,7 +1596,7 @@ async def create_project_with_parcel_data( # Check if Parzelle with this label already exists existing_parzellen = realEstateInterface.getParzellen( - recordFilter={"label": parcel_label, "mandateId": currentUser.mandateId} + recordFilter={"label": parcel_label, "mandateId": mandateId} ) if existing_parzellen and len(existing_parzellen) > 0: @@ -1630,7 +1639,7 @@ async def create_project_with_parcel_data( if not laender: logger.info("Creating Land 'Schweiz'") land = Land( - mandateId=currentUser.mandateId, + mandateId=mandateId, label="Schweiz", abk="CH" ) @@ -1648,7 +1657,7 @@ async def create_project_with_parcel_data( logger.info(f"Kanton '{canton_abk}' not found, creating it") kanton_label = canton_names.get(canton_abk, canton_abk) # Use mapping or fallback to abk kanton = Kanton( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=kanton_label, abk=canton_abk, id_land=land.id @@ -1668,7 +1677,7 @@ async def create_project_with_parcel_data( if not gemeinden: logger.info(f"Gemeinde '{municipality_name}' in Kanton '{canton_abk}' not found, creating it") gemeinde = Gemeinde( - mandateId=currentUser.mandateId, + mandateId=mandateId, label=municipality_name, id_kanton=kanton.id, plz=parzelle_data.get("plz") # Use PLZ directly from Swiss Topo API @@ -1837,7 +1846,7 @@ async def create_project_with_parcel_data( # Build Parzelle data parzelle_create_data = { - "mandateId": currentUser.mandateId, + "mandateId": mandateId, "label": parcel_label, # Use the label we determined earlier for uniqueness check "parzellenAliasTags": alias_tags, "eigentuemerschaft": None, @@ -1979,7 +1988,7 @@ async def create_project_with_parcel_data( project_perimeter = created_parzellen[0].perimeter if created_parzellen else None projekt_create_data = { - "mandateId": currentUser.mandateId, + "mandateId": mandateId, "label": projekt_label, "statusProzess": status_prozess_enum, "perimeter": project_perimeter, # Use first parcel perimeter as project perimeter diff --git a/modules/features/workflow/mainWorkflow.py b/modules/features/workflow/mainWorkflow.py index 08205fcc..70a2e9aa 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/features/workflow/mainWorkflow.py @@ -83,8 +83,8 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: executionLog["messages"].append(f"Started execution at {executionStartTime}") # 2. Replace placeholders in template to generate plan - template = automation.get("template", "") - placeholders = automation.get("placeholders", {}) + template = automation.template or "" + placeholders = automation.placeholders or {} planJson = replacePlaceholders(template, placeholders) try: plan = json.loads(planJson) @@ -102,7 +102,7 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: executionLog["messages"].append("Template placeholders replaced successfully") # 3. Get user who created automation - creatorUserId = automation.get("_createdBy") + creatorUserId = getattr(automation, "_createdBy", None) # CRITICAL: Automation MUST run as creator user only, or fail if not creatorUserId: @@ -147,13 +147,13 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)") # Set workflow name with "automated" prefix - automationLabel = automation.get("label", "Unknown Automation") + automationLabel = automation.label or "Unknown Automation" workflowName = f"automated: {automationLabel}" workflow = services.interfaceDbChat.updateWorkflow(workflow.id, {"name": workflowName}) logger.info(f"Set workflow {workflow.id} name to: {workflowName}") # Update automation with execution log - executionLogs = automation.get("executionLogs", []) + executionLogs = list(automation.executionLogs or []) executionLogs.append(executionLog) # Keep only last 50 executions if len(executionLogs) > 50: @@ -174,7 +174,7 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: try: automation = services.interfaceDbChat.getAutomationDefinition(automationId) if automation: - executionLogs = automation.get("executionLogs", []) + executionLogs = list(automation.executionLogs or []) executionLogs.append(executionLog) if len(executionLogs) > 50: executionLogs = executionLogs[-50:] @@ -204,10 +204,10 @@ async def syncAutomationEvents(services, eventUser) -> Dict[str, Any]: registeredEvents = {} for automation in filtered: - automationId = automation.get("id") - isActive = automation.get("active", False) - currentEventId = automation.get("eventId") - schedule = automation.get("schedule") + automationId = automation.id + isActive = automation.active if hasattr(automation, 'active') else False + currentEventId = automation.eventId if hasattr(automation, 'eventId') else None + schedule = automation.schedule if hasattr(automation, 'schedule') else None if not schedule: logger.warning(f"Automation {automationId} has no schedule, skipping") @@ -288,12 +288,12 @@ def createAutomationEventHandler(automationId: str, eventUser): # Load automation using event user context automation = eventServices.interfaceDbChat.getAutomationDefinition(automationId) - if not automation or not automation.get("active"): + if not automation or not getattr(automation, "active", False): logger.warning(f"Automation {automationId} not found or not active, skipping execution") return # Get creator user - creatorUserId = automation.get("_createdBy") + creatorUserId = getattr(automation, "_createdBy", None) if not creatorUserId: logger.error(f"Automation {automationId} has no creator user") return diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index f17f2cd7..95b660ec 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -3,10 +3,15 @@ """ Centralized bootstrap interface for system initialization. Contains all bootstrap logic including mandate, users, and RBAC rules. + +Multi-Tenant Design: +- Rollen werden mit Kontext erstellt (mandateId=None für globale Template-Rollen) +- AccessRules referenzieren roleId (FK), nicht roleLabel +- Admin-User bekommt isSysAdmin=True statt roleLabels """ import logging -from typing import Optional, List, Dict, Any +from typing import Optional, Dict from passlib.context import CryptContext from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG @@ -21,12 +26,19 @@ from modules.datamodels.datamodelRbac import ( Role, ) from modules.datamodels.datamodelUam import AccessLevel +from modules.datamodels.datamodelMembership import ( + UserMandate, + UserMandateRole, +) logger = logging.getLogger(__name__) # Password-Hashing pwdContext = CryptContext(schemes=["argon2"], deprecated="auto") +# Cache für Role-IDs (roleLabel -> roleId) +_roleIdCache: Dict[str, str] = {} + def initBootstrap(db: DatabaseConnector) -> None: """ @@ -40,21 +52,24 @@ def initBootstrap(db: DatabaseConnector) -> None: # Initialize root mandate mandateId = initRootMandate(db) + # Initialize roles FIRST (needed for AccessRules) + initRoles(db) + + # Initialize RBAC rules (uses roleIds from roles) + initRbacRules(db) + # Initialize admin user adminUserId = initAdminUser(db, mandateId) # Initialize event user eventUserId = initEventUser(db, mandateId) - # Initialize roles - initRoles(db) + # Assign initial user memberships (via UserMandate + UserMandateRole) + if adminUserId and eventUserId and mandateId: + assignInitialUserMemberships(db, mandateId, adminUserId, eventUserId) - # Initialize RBAC rules - initRbacRules(db) - - # Assign initial user roles - if adminUserId and eventUserId: - assignInitialUserRoles(db, adminUserId, eventUserId) + # Apply multi-tenant database optimizations (indexes, triggers, FKs) + _applyDatabaseOptimizations(db) logger.info("System bootstrap completed") @@ -76,7 +91,7 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]: return mandateId logger.info("Creating Root mandate") - rootMandate = Mandate(name="Root", language="en", enabled=True) + rootMandate = Mandate(name="Root", enabled=True) createdMandate = db.recordCreate(Mandate, rootMandate) mandateId = createdMandate.get("id") logger.info(f"Root mandate created with ID {mandateId}") @@ -86,10 +101,12 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]: def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[str]: """ Creates the Admin user if it doesn't exist. + Admin user gets isSysAdmin=True for system-level access. + Role assignment is done via UserMandate + UserMandateRole in assignInitialUserMemberships(). Args: db: Database connector instance - mandateId: Root mandate ID + mandateId: Root mandate ID (for membership assignment, not on User) Returns: User ID if created or found, None otherwise @@ -102,16 +119,14 @@ def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[s logger.info("Creating Admin user") adminUser = UserInDB( - mandateId=mandateId, username="admin", email="admin@example.com", fullName="Administrator", enabled=True, language="en", - roleLabels=["sysadmin"], + isSysAdmin=True, authenticationAuthority=AuthAuthority.LOCAL, hashedPassword=_getPasswordHash(APP_CONFIG.get("APP_INIT_PASS_ADMIN_SECRET")), - connections=[], ) createdUser = db.recordCreate(UserInDB, adminUser) userId = createdUser.get("id") @@ -122,10 +137,12 @@ def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[s def initEventUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[str]: """ Creates the Event user if it doesn't exist. + Event user gets isSysAdmin=True for system operations. + Role assignment is done via UserMandate + UserMandateRole in assignInitialUserMemberships(). Args: db: Database connector instance - mandateId: Root mandate ID + mandateId: Root mandate ID (for membership assignment, not on User) Returns: User ID if created or found, None otherwise @@ -138,16 +155,14 @@ def initEventUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[s logger.info("Creating Event user") eventUser = UserInDB( - mandateId=mandateId, username="event", email="event@example.com", fullName="Event", enabled=True, language="en", - roleLabels=["sysadmin"], + isSysAdmin=True, authenticationAuthority=AuthAuthority.LOCAL, hashedPassword=_getPasswordHash(APP_CONFIG.get("APP_INIT_PASS_EVENT_SECRET")), - connections=[], ) createdUser = db.recordCreate(UserInDB, eventUser) userId = createdUser.get("id") @@ -158,56 +173,99 @@ def initEventUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[s def initRoles(db: DatabaseConnector) -> None: """ Initialize standard roles if they don't exist. + Roles are created as GLOBAL (mandateId=None) template roles. Args: db: Database connector instance """ logger.info("Initializing roles") + global _roleIdCache + _roleIdCache = {} standardRoles = [ Role( roleLabel="sysadmin", - description={"en": "System Administrator - Full access to all system resources", "fr": "Administrateur système - Accès complet à toutes les ressources"}, + description={"en": "System Administrator - Full access to all system resources", "de": "System-Administrator - Vollzugriff auf alle System-Ressourcen", "fr": "Administrateur système - Accès complet à toutes les ressources"}, + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=None, isSystemRole=True ), Role( roleLabel="admin", - description={"en": "Administrator - Manage users and resources within mandate scope", "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat"}, + description={"en": "Administrator - Manage users and resources within mandate scope", "de": "Administrator - Benutzer und Ressourcen im Mandanten verwalten", "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat"}, + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=None, isSystemRole=True ), Role( roleLabel="user", - description={"en": "User - Standard user with access to own records", "fr": "Utilisateur - Utilisateur standard avec accès à ses propres enregistrements"}, + description={"en": "User - Standard user with access to own records", "de": "Benutzer - Standard-Benutzer mit Zugriff auf eigene Datensätze", "fr": "Utilisateur - Utilisateur standard avec accès à ses propres enregistrements"}, + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=None, isSystemRole=True ), Role( roleLabel="viewer", - description={"en": "Viewer - Read-only access to group records", "fr": "Visualiseur - Accès en lecture seule aux enregistrements du groupe"}, + description={"en": "Viewer - Read-only access to group records", "de": "Betrachter - Nur-Lese-Zugriff auf Gruppen-Datensätze", "fr": "Visualiseur - Accès en lecture seule aux enregistrements du groupe"}, + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=None, isSystemRole=True ), ] existingRoles = db.getRecordset(Role) - existingRoleLabels = {role.get("roleLabel") for role in existingRoles} + existingRoleLabels = {role.get("roleLabel"): role.get("id") for role in existingRoles} for role in standardRoles: if role.roleLabel not in existingRoleLabels: try: - db.recordCreate(Role, role) - logger.info(f"Created role: {role.roleLabel}") + createdRole = db.recordCreate(Role, role) + _roleIdCache[role.roleLabel] = createdRole.get("id") + logger.info(f"Created role: {role.roleLabel} with ID {createdRole.get('id')}") except Exception as e: logger.warning(f"Error creating role {role.roleLabel}: {e}") else: - logger.debug(f"Role {role.roleLabel} already exists") + _roleIdCache[role.roleLabel] = existingRoleLabels[role.roleLabel] + logger.debug(f"Role {role.roleLabel} already exists with ID {existingRoleLabels[role.roleLabel]}") logger.info("Roles initialization completed") +def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: + """ + Get role ID by label, using cache or database lookup. + + Args: + db: Database connector + roleLabel: Role label to look up + + Returns: + Role ID or None if not found + """ + global _roleIdCache + + if roleLabel in _roleIdCache: + return _roleIdCache[roleLabel] + + # Lookup from database + roles = db.getRecordset(Role, recordFilter={"roleLabel": roleLabel}) + if roles: + roleId = roles[0].get("id") + _roleIdCache[roleLabel] = roleId + return roleId + + logger.warning(f"Role not found: {roleLabel}") + return None + + def initRbacRules(db: DatabaseConnector) -> None: """ Initialize RBAC rules if they don't exist. - Converts all UAM logic from interface*Access.py modules to RBAC rules. - Also checks for and adds missing rules for new tables. + AccessRules now reference roleId (FK) instead of roleLabel. Args: db: Database connector instance @@ -215,41 +273,40 @@ def initRbacRules(db: DatabaseConnector) -> None: existingRules = db.getRecordset(AccessRule) if existingRules: logger.info(f"RBAC rules already exist ({len(existingRules)} rules)") - # Check for missing rules for ChatWorkflow and Prompt tables - _addMissingTableRules(db, existingRules) return logger.info("Initializing RBAC rules") # Create default role rules - createDefaultRoleRules(db) + _createDefaultRoleRules(db) # Create table-specific rules (converted from UAM logic) - createTableSpecificRules(db) + _createTableSpecificRules(db) # Create UI context rules - createUiContextRules(db) + _createUiContextRules(db) # Create RESOURCE context rules - createResourceContextRules(db) - - # Create Action-specific RBAC rules - createActionRules(db) + _createResourceContextRules(db) logger.info("RBAC rules initialization completed") -def createDefaultRoleRules(db: DatabaseConnector) -> None: +def _createDefaultRoleRules(db: DatabaseConnector) -> None: """ Create default role rules for generic access (item = null). + Uses roleId instead of roleLabel. Args: db: Database connector instance """ - defaultRules = [ - # SysAdmin Role - Full access to all - AccessRule( - roleLabel="sysadmin", + defaultRules = [] + + # SysAdmin Role - Full access to all + sysadminId = _getRoleId(db, "sysadmin") + if sysadminId: + defaultRules.append(AccessRule( + roleId=sysadminId, context=AccessRuleContext.DATA, item=None, view=True, @@ -257,10 +314,13 @@ def createDefaultRoleRules(db: DatabaseConnector) -> None: create=AccessLevel.ALL, update=AccessLevel.ALL, delete=AccessLevel.ALL, - ), - # Admin Role - Group-level access - AccessRule( - roleLabel="admin", + )) + + # Admin Role - Group-level access + adminId = _getRoleId(db, "admin") + if adminId: + defaultRules.append(AccessRule( + roleId=adminId, context=AccessRuleContext.DATA, item=None, view=True, @@ -268,10 +328,13 @@ def createDefaultRoleRules(db: DatabaseConnector) -> None: create=AccessLevel.GROUP, update=AccessLevel.GROUP, delete=AccessLevel.NONE, - ), - # User Role - My records only - AccessRule( - roleLabel="user", + )) + + # User Role - My records only + userId = _getRoleId(db, "user") + if userId: + defaultRules.append(AccessRule( + roleId=userId, context=AccessRuleContext.DATA, item=None, view=True, @@ -279,10 +342,13 @@ def createDefaultRoleRules(db: DatabaseConnector) -> None: create=AccessLevel.MY, update=AccessLevel.MY, delete=AccessLevel.MY, - ), - # Viewer Role - Read-only group access - AccessRule( - roleLabel="viewer", + )) + + # Viewer Role - Read-only group access + viewerId = _getRoleId(db, "viewer") + if viewerId: + defaultRules.append(AccessRule( + roleId=viewerId, context=AccessRuleContext.DATA, item=None, view=True, @@ -290,8 +356,7 @@ def createDefaultRoleRules(db: DatabaseConnector) -> None: create=AccessLevel.NONE, update=AccessLevel.NONE, delete=AccessLevel.NONE, - ), - ] + )) for rule in defaultRules: db.recordCreate(AccessRule, rule) @@ -299,393 +364,209 @@ def createDefaultRoleRules(db: DatabaseConnector) -> None: logger.info(f"Created {len(defaultRules)} default role rules") -def createTableSpecificRules(db: DatabaseConnector) -> None: +def _createTableSpecificRules(db: DatabaseConnector) -> None: """ Create table-specific rules converted from UAM logic. These rules override generic rules for specific tables. + Uses roleId instead of roleLabel. Args: db: Database connector instance """ tableRules = [] + # Get role IDs + sysadminId = _getRoleId(db, "sysadmin") + adminId = _getRoleId(db, "admin") + userId = _getRoleId(db, "user") + viewerId = _getRoleId(db, "viewer") + # Mandate table - Only sysadmin can access - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="Mandate", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="Mandate", - view=False, - read=AccessLevel.NONE, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="Mandate", - view=False, - read=AccessLevel.NONE, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="Mandate", - view=False, - read=AccessLevel.NONE, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # UserInDB table - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="UserInDB", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="UserInDB", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="UserInDB", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.MY, - delete=AccessLevel.NONE, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="UserInDB", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # UserConnection table - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="UserConnection", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="UserConnection", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="UserConnection", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="UserConnection", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # DataNeutraliserConfig table - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="DataNeutraliserConfig", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="DataNeutraliserConfig", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="DataNeutraliserConfig", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="DataNeutraliserConfig", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # DataNeutralizerAttributes table - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="DataNeutralizerAttributes", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="DataNeutralizerAttributes", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="DataNeutralizerAttributes", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="DataNeutralizerAttributes", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # AuthEvent table - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="AuthEvent", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="AuthEvent", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="AuthEvent", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="AuthEvent", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # ChatWorkflow table - Users can access their own workflows - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="ChatWorkflow", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="ChatWorkflow", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="ChatWorkflow", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="ChatWorkflow", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # Prompt table - Users can access their own prompts - tableRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.DATA, - item="Prompt", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - tableRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.DATA, - item="Prompt", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - tableRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.DATA, - item="Prompt", - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - tableRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.DATA, - item="Prompt", - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # Real Estate tables - Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land - realEstateTables = ["Projekt", "Parzelle", "Dokument", "Gemeinde", "Kanton", "Land"] - for table in realEstateTables: - # Sysadmin - full access + if sysadminId: tableRules.append(AccessRule( - roleLabel="sysadmin", + roleId=sysadminId, context=AccessRuleContext.DATA, - item=table, + item="Mandate", view=True, read=AccessLevel.ALL, create=AccessLevel.ALL, update=AccessLevel.ALL, delete=AccessLevel.ALL, )) - # Admin - group access + if adminId: tableRules.append(AccessRule( - roleLabel="admin", + roleId=adminId, context=AccessRuleContext.DATA, - item=table, + item="Mandate", + view=False, + read=AccessLevel.NONE, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + if userId: + tableRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item="Mandate", + view=False, + read=AccessLevel.NONE, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="Mandate", + view=False, + read=AccessLevel.NONE, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # UserInDB table + if sysadminId: + tableRules.append(AccessRule( + roleId=sysadminId, + context=AccessRuleContext.DATA, + item="UserInDB", + view=True, + read=AccessLevel.ALL, + create=AccessLevel.ALL, + update=AccessLevel.ALL, + delete=AccessLevel.ALL, + )) + if adminId: + tableRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item="UserInDB", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, update=AccessLevel.GROUP, delete=AccessLevel.GROUP, )) - # User - my records only + if userId: tableRules.append(AccessRule( - roleLabel="user", + roleId=userId, context=AccessRuleContext.DATA, - item=table, + item="UserInDB", view=True, read=AccessLevel.MY, - create=AccessLevel.MY, + create=AccessLevel.NONE, update=AccessLevel.MY, - delete=AccessLevel.MY, + delete=AccessLevel.NONE, )) - # Viewer - read-only my records + if viewerId: tableRules.append(AccessRule( - roleLabel="viewer", + roleId=viewerId, context=AccessRuleContext.DATA, - item=table, + item="UserInDB", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # Standard tables with typical access patterns + standardTables = [ + "UserConnection", "DataNeutraliserConfig", "DataNeutralizerAttributes", + "ChatWorkflow", "Prompt", "Projekt", "Parzelle", "Dokument", + "Gemeinde", "Kanton", "Land", + "TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", "TrusteeContract", + "TrusteeDocument", "TrusteePosition", "TrusteePositionDocument" + ] + + for table in standardTables: + if sysadminId: + tableRules.append(AccessRule( + roleId=sysadminId, + context=AccessRuleContext.DATA, + item=table, + view=True, + read=AccessLevel.ALL, + create=AccessLevel.ALL, + update=AccessLevel.ALL, + delete=AccessLevel.ALL, + )) + if adminId: + tableRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item=table, + view=True, + read=AccessLevel.GROUP, + create=AccessLevel.GROUP, + update=AccessLevel.GROUP, + delete=AccessLevel.GROUP, + )) + if userId: + tableRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item=table, + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item=table, + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # AuthEvent table - special handling + if sysadminId: + tableRules.append(AccessRule( + roleId=sysadminId, + context=AccessRuleContext.DATA, + item="AuthEvent", + view=True, + read=AccessLevel.ALL, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.ALL, + )) + if adminId: + tableRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item="AuthEvent", + view=True, + read=AccessLevel.ALL, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.ALL, + )) + if userId: + tableRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item="AuthEvent", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="AuthEvent", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -700,509 +581,124 @@ def createTableSpecificRules(db: DatabaseConnector) -> None: logger.info(f"Created {len(tableRules)} table-specific rules") -def createUiContextRules(db: DatabaseConnector) -> None: +def _createUiContextRules(db: DatabaseConnector) -> None: """ Create UI context rules for controlling UI element visibility. - These rules control which UI components users can see based on their roles. + Uses roleId instead of roleLabel. Args: db: Database connector instance """ uiRules = [] - # Generic UI rules - all roles can view UI by default - # Specific UI elements can override these with more restrictive rules + # All roles get full UI access by default + for roleLabel in ["sysadmin", "admin", "user", "viewer"]: + roleId = _getRoleId(db, roleLabel) + if roleId: + uiRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.UI, + item=None, + view=True, + read=None, + create=None, + update=None, + delete=None, + )) - # Sysadmin - full UI access - uiRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.UI, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Admin - full UI access - uiRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.UI, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # User - full UI access - uiRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.UI, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Viewer - full UI access (can view but may have restricted actions) - uiRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.UI, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Create all UI context rules for rule in uiRules: db.recordCreate(AccessRule, rule) logger.info(f"Created {len(uiRules)} UI context rules") -def createResourceContextRules(db: DatabaseConnector) -> None: +def _createResourceContextRules(db: DatabaseConnector) -> None: """ - Create RESOURCE context rules for controlling resource access (AI models, actions, etc.). - These rules control which resources users can access based on their roles. + Create RESOURCE context rules for controlling resource access. + Uses roleId instead of roleLabel. Args: db: Database connector instance """ resourceRules = [] - # Generic resource rules - all roles can access resources by default - # Specific resources can override these with more restrictive rules + # All roles get full resource access by default + for roleLabel in ["sysadmin", "admin", "user", "viewer"]: + roleId = _getRoleId(db, roleLabel) + if roleId: + resourceRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.RESOURCE, + item=None, + view=True, + read=None, + create=None, + update=None, + delete=None, + )) - # Sysadmin - full resource access - resourceRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.RESOURCE, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Admin - full resource access - resourceRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.RESOURCE, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # User - full resource access - resourceRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.RESOURCE, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Viewer - full resource access (can view but may have restricted actions) - resourceRules.append(AccessRule( - roleLabel="viewer", - context=AccessRuleContext.RESOURCE, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) - - # Create all RESOURCE context rules for rule in resourceRules: db.recordCreate(AccessRule, rule) logger.info(f"Created {len(resourceRules)} RESOURCE context rules") -def createActionRules(db: DatabaseConnector) -> None: +def assignInitialUserMemberships( + db: DatabaseConnector, + mandateId: str, + adminUserId: str, + eventUserId: str +) -> None: """ - Create default RBAC rules for workflow actions. - - This function dynamically discovers all available actions from all methods - and creates RBAC rules for them. Actions are protected via RESOURCE context - with actionId as the item identifier (format: 'module.actionName'). - - Args: - db: Database connector instance - """ - try: - # Import method discovery to get all actions - from modules.workflows.processing.shared.methodDiscovery import discoverMethods - from modules.services import getInterface as getServices - from modules.datamodels.datamodelUam import User - - # Create a temporary user context for discovery (will be filtered by RBAC later) - # We need to discover methods, but we'll use a minimal user context - # In production, this should use a system user or admin user - try: - # Try to get an admin user for discovery - adminUsers = db.getRecordset("User", recordFilter={"roleLabel": "sysadmin"}, limit=1) - if adminUsers: - tempUser = User(**adminUsers[0]) - else: - # Fallback: create minimal user context - tempUser = User(id="system", roleLabel="sysadmin") - except: - # Fallback: create minimal user context - tempUser = User(id="system", roleLabel="sysadmin") - - # Get services and discover methods - services = getServices(tempUser, None) - discoverMethods(services) - - # Import methods catalog - from modules.workflows.processing.shared.methodDiscovery import methods - - # Collect all action IDs - allActionIds = [] - for methodName, methodInfo in methods.items(): - # Skip duplicate entries (same method stored with full and short name) - if methodName.startswith('Method'): - continue - - methodInstance = methodInfo['instance'] - methodActions = methodInstance.actions - - for actionName in methodActions.keys(): - actionId = f"{methodInstance.name}.{actionName}" - allActionIds.append(actionId) - - logger.info(f"Discovered {len(allActionIds)} actions for RBAC rule creation") - - # Define default action access by role - # SysAdmin and Admin: Access to all actions - # User: Access to common actions (read, search, process, etc.) - # Viewer: Read-only actions - - actionRules = [] - - # All roles: Generic access to all actions - # Using item=None grants access to all resources (all actions) in RESOURCE context - - # SysAdmin: Access to all actions - actionRules.append(AccessRule( - roleLabel="sysadmin", - context=AccessRuleContext.RESOURCE, - item=None, # All resources (covers all actions) - view=True - )) - - # Admin: Access to all actions - actionRules.append(AccessRule( - roleLabel="admin", - context=AccessRuleContext.RESOURCE, - item=None, # All resources (covers all actions) - view=True - )) - - # User: Access to all actions (generic rights) - actionRules.append(AccessRule( - roleLabel="user", - context=AccessRuleContext.RESOURCE, - item=None, # All resources (covers all actions) - view=True - )) - - - # Create all action rules - for rule in actionRules: - db.recordCreate(AccessRule, rule) - - logger.info(f"Created {len(actionRules)} action RBAC rules") - - except Exception as e: - logger.error(f"Error creating action RBAC rules: {str(e)}", exc_info=True) - # Don't fail bootstrap if action rules can't be created - # They can be created manually or via migration script - - -def _addMissingTableRules(db: DatabaseConnector, existingRules: List[Dict[str, Any]]) -> None: - """ - Add missing RBAC rules for tables that were added after initial bootstrap. - - Args: - db: Database connector instance - existingRules: List of existing AccessRule records - """ - # Check which tables already have rules - existingItems = {rule.get("item") for rule in existingRules if rule.get("context") == AccessRuleContext.DATA} - existingRoles = {rule.get("roleLabel") for rule in existingRules} - - # Tables that need rules - requiredTables = [ - "ChatWorkflow", "Prompt", "Projekt", "Parzelle", "Dokument", "Gemeinde", "Kanton", "Land", - # Trustee tables - "TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", "TrusteeContract", - "TrusteeDocument", "TrusteePosition", "TrusteePositionDocument" - ] - requiredRoles = ["sysadmin", "admin", "user", "viewer"] - - newRules = [] - - for table in requiredTables: - if table not in existingItems: - logger.info(f"Adding missing RBAC rules for table {table}") - # ChatWorkflow rules - if table == "ChatWorkflow": - for roleLabel in requiredRoles: - if roleLabel == "sysadmin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - elif roleLabel == "admin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - elif roleLabel == "user": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - elif roleLabel == "viewer": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - # Prompt rules (same as ChatWorkflow) - elif table == "Prompt": - for roleLabel in requiredRoles: - if roleLabel == "sysadmin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - elif roleLabel == "admin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - elif roleLabel == "user": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - elif roleLabel == "viewer": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - # Real Estate tables rules (Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land) - elif table in ["Projekt", "Parzelle", "Dokument", "Gemeinde", "Kanton", "Land"]: - for roleLabel in requiredRoles: - if roleLabel == "sysadmin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - elif roleLabel == "admin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - elif roleLabel == "user": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - elif roleLabel == "viewer": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - # Trustee tables rules - elif table in ["TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", "TrusteeContract", - "TrusteeDocument", "TrusteePosition", "TrusteePositionDocument"]: - for roleLabel in requiredRoles: - if roleLabel == "sysadmin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - elif roleLabel == "admin": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, - )) - elif roleLabel == "user": - # User role: Access to MY records (feature-specific access via trustee.access) - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - elif roleLabel == "viewer": - newRules.append(AccessRule( - roleLabel=roleLabel, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # Create missing rules - if newRules: - for rule in newRules: - db.recordCreate(AccessRule, rule) - logger.info(f"Added {len(newRules)} missing RBAC rules") - - -def assignInitialUserRoles(db: DatabaseConnector, adminUserId: str, eventUserId: str) -> None: - """ - Assign initial roles to admin and event users. + Assign initial memberships to admin and event users via UserMandate + UserMandateRole. + This is the NEW multi-tenant way of assigning roles. Args: db: Database connector instance + mandateId: Root mandate ID adminUserId: Admin user ID eventUserId: Event user ID """ - # Set context to admin user for bootstrap operations - originalUserId = db.userId if hasattr(db, 'userId') else None - try: - if adminUserId: - db.updateContext(adminUserId) + sysadminRoleId = _getRoleId(db, "sysadmin") + if not sysadminRoleId: + logger.warning("Sysadmin role not found, skipping membership assignment") + return + + for userId, userName in [(adminUserId, "admin"), (eventUserId, "event")]: + # Check if UserMandate already exists + existingMemberships = db.getRecordset( + UserMandate, + recordFilter={"userId": userId, "mandateId": mandateId} + ) - # Update admin user with sysadmin role - adminUser = db.getRecordset(UserInDB, recordFilter={"id": adminUserId}) - if adminUser: - adminUserData = adminUser[0] - roleLabels = adminUserData.get("roleLabels") or [] - if "sysadmin" not in roleLabels: - adminUserData["roleLabels"] = roleLabels + ["sysadmin"] - db.recordModify(UserInDB, adminUserId, adminUserData) - logger.info(f"Assigned sysadmin role to admin user {adminUserId}") + if existingMemberships: + userMandateId = existingMemberships[0].get("id") + logger.debug(f"UserMandate already exists for {userName} user") + else: + # Create UserMandate + userMandate = UserMandate( + userId=userId, + mandateId=mandateId, + enabled=True + ) + createdMembership = db.recordCreate(UserMandate, userMandate) + userMandateId = createdMembership.get("id") + logger.info(f"Created UserMandate for {userName} user with ID {userMandateId}") - # Update event user with sysadmin role - eventUser = db.getRecordset(UserInDB, recordFilter={"id": eventUserId}) - if eventUser: - eventUserData = eventUser[0] - roleLabels = eventUserData.get("roleLabels") or [] - if "sysadmin" not in roleLabels: - eventUserData["roleLabels"] = roleLabels + ["sysadmin"] - db.recordModify(UserInDB, eventUserId, eventUserData) - logger.info(f"Assigned sysadmin role to event user {eventUserId}") - finally: - # Restore original context if it existed - if originalUserId: - db.updateContext(originalUserId) - elif hasattr(db, 'userId'): - # If original was None/empty, just set it directly - db.userId = originalUserId + # Check if UserMandateRole already exists + existingRoles = db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId, "roleId": sysadminRoleId} + ) + + if not existingRoles: + # Create UserMandateRole + userMandateRole = UserMandateRole( + userMandateId=userMandateId, + roleId=sysadminRoleId + ) + db.recordCreate(UserMandateRole, userMandateRole) + logger.info(f"Assigned sysadmin role to {userName} user in mandate") def _getPasswordHash(password: Optional[str]) -> Optional[str]: @@ -1218,3 +714,42 @@ def _getPasswordHash(password: Optional[str]) -> Optional[str]: if password is None: return None return pwdContext.hash(password) + + +def _applyDatabaseOptimizations(db: DatabaseConnector) -> None: + """ + Apply multi-tenant database optimizations after bootstrap. + + Creates indexes, immutable triggers, and foreign key constraints + for the multi-tenant junction tables. All operations are idempotent. + + Args: + db: Database connector instance + """ + try: + from modules.shared.dbMultiTenantOptimizations import applyMultiTenantOptimizations + + result = applyMultiTenantOptimizations(db) + + if result.get("errors"): + for error in result["errors"]: + logger.warning(f"DB optimization error: {error}") + else: + totalCreated = ( + result.get("indexesCreated", 0) + + result.get("triggersCreated", 0) + + result.get("foreignKeysCreated", 0) + ) + if totalCreated > 0: + logger.info( + f"Applied DB optimizations: {result['indexesCreated']} indexes, " + f"{result['triggersCreated']} triggers, " + f"{result['foreignKeysCreated']} foreign keys" + ) + # If nothing created, optimizations were already applied (idempotent) + + except ImportError as e: + logger.warning(f"DB optimizations module not available: {e}") + except Exception as e: + # Don't fail bootstrap if optimizations fail + logger.warning(f"Failed to apply DB optimizations (non-critical): {e}") diff --git a/modules/interfaces/interfaceDbAppObjects.py b/modules/interfaces/interfaceDbAppObjects.py index ab792b08..d661e603 100644 --- a/modules/interfaces/interfaceDbAppObjects.py +++ b/modules/interfaces/interfaceDbAppObjects.py @@ -3,6 +3,10 @@ """ Interface to the Gateway system. Manages users and mandates for authentication. + +Multi-Tenant Design: +- User gehört nicht mehr direkt zu einem Mandanten +- mandateId wird aus Request-Context übergeben (X-Mandate-Id Header) """ import logging @@ -37,6 +41,14 @@ from modules.datamodels.datamodelNeutralizer import ( DataNeutralizerAttributes, ) from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult +from modules.datamodels.datamodelMembership import ( + UserMandate, + UserMandateRole, + FeatureAccess, + FeatureAccessRole, +) +from modules.datamodels.datamodelFeatures import Feature, FeatureInstance +from modules.datamodels.datamodelInvitation import Invitation logger = logging.getLogger(__name__) @@ -61,7 +73,7 @@ class AppObjects: # Initialize variables self.currentUser = currentUser # Store User object directly self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None + self.mandateId = None # mandateId comes from setUserContext, not from User # Initialize database self._initializeDatabase() @@ -73,25 +85,42 @@ class AppObjects: if currentUser: self.setUserContext(currentUser) - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """ + Sets the user context for the interface. + + Multi-Tenant Design: + - mandateId wird explizit übergeben (aus Request-Context / X-Mandate-Id Header) + - isSysAdmin User brauchen kein mandateId für System-Operationen + + Args: + currentUser: User object + mandateId: Explicit mandate context (from request header). Required for non-sysadmin. + """ if not currentUser: logger.info("Initializing interface without user context") return self.currentUser = currentUser # Store User object directly self.userId = currentUser.id - self.mandateId = currentUser.mandateId + + # mandateId comes from parameter only + self.mandateId = mandateId - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") + # Validate: userId is always required + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId is optional for isSysAdmin users doing system-level operations + isSysAdmin = getattr(currentUser, 'isSysAdmin', False) + if not self.mandateId and not isSysAdmin: + # Non-sysadmin users MUST have a mandateId for tenant-scoped operations + logger.warning(f"User {self.userId} has no mandateId context") # Add language settings self.userLanguage = currentUser.language # Default user language # Initialize RBAC interface - if not currentUser: - raise ValueError("User context is required for RBAC") # Pass self.db as dbApp since this interface uses DbApp database self.rbac = RbacClass(self.db, dbApp=self.db) @@ -110,11 +139,11 @@ class AppObjects: """Initializes the database connection directly.""" try: # Get configuration values with defaults - dbHost = APP_CONFIG.get("DB_APP_HOST", "_no_config_default_data") - dbDatabase = APP_CONFIG.get("DB_APP_DATABASE", "app") - dbUser = APP_CONFIG.get("DB_APP_USER") - dbPassword = APP_CONFIG.get("DB_APP_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_APP_PORT", 5432)) + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_app" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) # Create database connector directly self.db = DatabaseConnector( @@ -615,13 +644,17 @@ class AppObjects: fullName: str = None, language: str = "en", enabled: bool = True, - roleLabels: List[str] = None, authenticationAuthority: AuthAuthority = AuthAuthority.LOCAL, externalId: str = None, externalUsername: str = None, externalEmail: str = None, + isSysAdmin: bool = False, ) -> User: - """Create a new user with optional external connection""" + """ + Create a new user. + + Note: Role assignment is done via createUserMandate(), not via User fields. + """ try: # Ensure username is a string username = str(username).strip() @@ -638,28 +671,17 @@ class AppObjects: if not password.strip(): raise ValueError("Password cannot be empty") - # Ensure mandateId is set - use self.mandateId or default mandate - mandateId = self.mandateId - if not mandateId: - mandateId = self._getDefaultMandateId() - logger.warning(f"Using default mandate ID {mandateId} for new user {username}") - - # Default roleLabels to ["user"] if not provided - if roleLabels is None or not roleLabels: - roleLabels = ["user"] - # Create user data using UserInDB model + # Note: mandateId and roleLabels are REMOVED - use UserMandate + UserMandateRole userData = UserInDB( username=username, email=email, fullName=fullName, language=language, - mandateId=mandateId, enabled=enabled, - roleLabels=roleLabels, + isSysAdmin=isSysAdmin, authenticationAuthority=authenticationAuthority, hashedPassword=self._getPasswordHash(password) if password else None, - connections=[], ) # Create user record @@ -712,25 +734,11 @@ class AppObjects: # Remove id field from updateDict if present - we'll use userId from parameter updateDict.pop("id", None) - # Ensure mandateId is set - if missing or None, use default mandate - if "mandateId" not in updateDict or not updateDict.get("mandateId"): - if not user.mandateId: - # User has no mandateId, set to default - defaultMandateId = self._getDefaultMandateId() - updateDict["mandateId"] = defaultMandateId - logger.warning(f"Setting default mandate ID {defaultMandateId} for user {userId}") - else: - # Keep existing mandateId if update doesn't provide one - updateDict["mandateId"] = user.mandateId - # Update user data using model updatedData = user.model_dump() updatedData.update(updateDict) # Ensure ID matches userId parameter updatedData["id"] = userId - # Ensure mandateId is set in final data - if not updatedData.get("mandateId"): - updatedData["mandateId"] = self._getDefaultMandateId() updatedUser = User(**updatedData) # Update user record @@ -1382,6 +1390,325 @@ class AppObjects: logger.error(f"Error deleting mandate: {str(e)}") raise ValueError(f"Failed to delete mandate: {str(e)}") + # ============================================ + # User-Mandate Membership Methods (Multi-Tenant) + # ============================================ + + def getUserMandate(self, userId: str, mandateId: str) -> Optional[UserMandate]: + """ + Get UserMandate record for a user in a specific mandate. + + Args: + userId: User ID + mandateId: Mandate ID + + Returns: + UserMandate object or None + """ + try: + records = self.db.getRecordset( + UserMandate, + recordFilter={"userId": userId, "mandateId": mandateId} + ) + if not records: + return None + cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + return UserMandate(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting UserMandate: {e}") + return None + + def getUserMandates(self, userId: str) -> List[UserMandate]: + """ + Get all mandates a user is member of. + + Args: + userId: User ID + + Returns: + List of UserMandate objects + """ + try: + records = self.db.getRecordset( + UserMandate, + recordFilter={"userId": userId, "enabled": True} + ) + result = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + result.append(UserMandate(**cleanedRecord)) + return result + except Exception as e: + logger.error(f"Error getting UserMandates: {e}") + return [] + + def createUserMandate(self, userId: str, mandateId: str, roleIds: List[str] = None) -> UserMandate: + """ + Create a UserMandate record (add user to mandate). + + Args: + userId: User ID + mandateId: Mandate ID + roleIds: Optional list of role IDs to assign + + Returns: + Created UserMandate object + """ + try: + # Check if already exists + existing = self.getUserMandate(userId, mandateId) + if existing: + raise ValueError(f"User {userId} is already member of mandate {mandateId}") + + # Create UserMandate + userMandate = UserMandate( + userId=userId, + mandateId=mandateId, + enabled=True + ) + createdRecord = self.db.recordCreate(UserMandate, userMandate.model_dump()) + + # Assign roles via junction table + if roleIds and createdRecord: + userMandateId = createdRecord.get("id") + for roleId in roleIds: + userMandateRole = UserMandateRole( + userMandateId=userMandateId, + roleId=roleId + ) + self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) + + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + return UserMandate(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating UserMandate: {e}") + raise ValueError(f"Failed to create UserMandate: {e}") + + def deleteUserMandate(self, userId: str, mandateId: str) -> bool: + """ + Delete a UserMandate record (remove user from mandate). + CASCADE will delete UserMandateRole entries. + + Args: + userId: User ID + mandateId: Mandate ID + + Returns: + True if deleted, False if not found + """ + try: + existing = self.getUserMandate(userId, mandateId) + if not existing: + return False + + return self.db.recordDelete(UserMandate, existing.id) + except Exception as e: + logger.error(f"Error deleting UserMandate: {e}") + raise ValueError(f"Failed to delete UserMandate: {e}") + + def getRoleIdsForUserMandate(self, userMandateId: str) -> List[str]: + """ + Get all role IDs assigned to a UserMandate. + + Args: + userMandateId: UserMandate ID + + Returns: + List of role IDs + """ + try: + records = self.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId} + ) + return [r.get("roleId") for r in records if r.get("roleId")] + except Exception as e: + logger.error(f"Error getting role IDs for UserMandate: {e}") + return [] + + def addRoleToUserMandate(self, userMandateId: str, roleId: str) -> UserMandateRole: + """ + Add a role to a UserMandate. + + Args: + userMandateId: UserMandate ID + roleId: Role ID to add + + Returns: + Created UserMandateRole object + """ + try: + # Check if already exists + existing = self.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId, "roleId": roleId} + ) + if existing: + cleanedRecord = {k: v for k, v in existing[0].items() if not k.startswith("_")} + return UserMandateRole(**cleanedRecord) + + userMandateRole = UserMandateRole( + userMandateId=userMandateId, + roleId=roleId + ) + createdRecord = self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + return UserMandateRole(**cleanedRecord) + except Exception as e: + logger.error(f"Error adding role to UserMandate: {e}") + raise ValueError(f"Failed to add role: {e}") + + def removeRoleFromUserMandate(self, userMandateId: str, roleId: str) -> bool: + """ + Remove a role from a UserMandate. + If no roles remain, the UserMandate is deleted (Application-Level Cleanup). + + Args: + userMandateId: UserMandate ID + roleId: Role ID to remove + + Returns: + True if removed + """ + try: + # Find and delete the junction record + records = self.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId, "roleId": roleId} + ) + if not records: + return False + + self.db.recordDelete(UserMandateRole, records[0].get("id")) + + # Application-Level Cleanup: Delete UserMandate if no roles remain + remainingRoles = self.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId} + ) + if not remainingRoles: + self.db.recordDelete(UserMandate, userMandateId) + logger.info(f"Deleted empty UserMandate {userMandateId}") + + return True + except Exception as e: + logger.error(f"Error removing role from UserMandate: {e}") + raise ValueError(f"Failed to remove role: {e}") + + # ============================================ + # Feature Access Methods (Multi-Tenant) + # ============================================ + + def getFeatureAccess(self, userId: str, featureInstanceId: str) -> Optional[FeatureAccess]: + """ + Get FeatureAccess record for a user to a specific feature instance. + + Args: + userId: User ID + featureInstanceId: FeatureInstance ID + + Returns: + FeatureAccess object or None + """ + try: + records = self.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": featureInstanceId} + ) + if not records: + return None + cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + return FeatureAccess(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting FeatureAccess: {e}") + return None + + def getFeatureAccessesForUser(self, userId: str) -> List[FeatureAccess]: + """ + Get all feature accesses for a user. + + Args: + userId: User ID + + Returns: + List of FeatureAccess objects + """ + try: + records = self.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "enabled": True} + ) + result = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + result.append(FeatureAccess(**cleanedRecord)) + return result + except Exception as e: + logger.error(f"Error getting FeatureAccesses: {e}") + return [] + + def createFeatureAccess(self, userId: str, featureInstanceId: str, roleIds: List[str] = None) -> FeatureAccess: + """ + Create a FeatureAccess record (grant user access to feature instance). + + Args: + userId: User ID + featureInstanceId: FeatureInstance ID + roleIds: Optional list of role IDs to assign + + Returns: + Created FeatureAccess object + """ + try: + # Check if already exists + existing = self.getFeatureAccess(userId, featureInstanceId) + if existing: + raise ValueError(f"User {userId} already has access to feature instance {featureInstanceId}") + + # Create FeatureAccess + featureAccess = FeatureAccess( + userId=userId, + featureInstanceId=featureInstanceId, + enabled=True + ) + createdRecord = self.db.recordCreate(FeatureAccess, featureAccess.model_dump()) + + # Assign roles via junction table + if roleIds and createdRecord: + featureAccessId = createdRecord.get("id") + for roleId in roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + self.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + return FeatureAccess(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating FeatureAccess: {e}") + raise ValueError(f"Failed to create FeatureAccess: {e}") + + def getRoleIdsForFeatureAccess(self, featureAccessId: str) -> List[str]: + """ + Get all role IDs assigned to a FeatureAccess. + + Args: + featureAccessId: FeatureAccess ID + + Returns: + List of role IDs + """ + try: + records = self.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + return [r.get("roleId") for r in records if r.get("roleId")] + except Exception as e: + logger.error(f"Error getting role IDs for FeatureAccess: {e}") + return [] + # Token methods def saveAccessToken(self, token: Token, replace_existing: bool = True) -> None: @@ -1902,6 +2229,7 @@ class AppObjects: def getAccessRules( self, roleLabel: Optional[str] = None, + roleId: Optional[str] = None, context: Optional[AccessRuleContext] = None, item: Optional[str] = None, pagination: Optional[PaginationParams] = None @@ -1910,7 +2238,8 @@ class AppObjects: Get access rules with optional filters and pagination. Args: - roleLabel: Optional role label filter + roleLabel: Optional role label filter (deprecated, use roleId) + roleId: Optional role ID filter context: Optional context filter item: Optional item filter pagination: Optional pagination parameters. If None, returns all items. @@ -1921,7 +2250,9 @@ class AppObjects: """ try: recordFilter = {} - if roleLabel: + if roleId: + recordFilter["roleId"] = roleId + elif roleLabel: recordFilter["roleLabel"] = roleLabel if context: recordFilter["context"] = context.value @@ -2134,6 +2465,29 @@ class AppObjects: else: return PaginatedResult(items=[], totalItems=0, totalPages=0) + def countRoleAssignments(self) -> Dict[str, int]: + """ + Count the number of user assignments per role from UserMandateRole table. + + Returns: + Dictionary mapping roleId to count of user assignments + """ + try: + # Get all UserMandateRole records + assignments = self.db.getRecordset(UserMandateRole) + + # Count assignments per roleId + roleCounts: Dict[str, int] = {} + for assignment in assignments: + roleId = str(assignment.get("roleId", "")) + if roleId: + roleCounts[roleId] = roleCounts.get(roleId, 0) + 1 + + return roleCounts + except Exception as e: + logger.error(f"Error counting role assignments: {str(e)}") + return {} + def updateRole(self, roleId: str, role: Role) -> Role: """ Update an existing role. @@ -2185,14 +2539,13 @@ class AppObjects: if role.isSystemRole: raise ValueError(f"Cannot delete system role '{role.roleLabel}'") - # Check if role is assigned to any users - allUsers = self.getUsersByMandate(None) # Get all users across all mandates - for user in allUsers: - if role.roleLabel in (user.roleLabels or []): - raise ValueError(f"Cannot delete role '{role.roleLabel}' - it is assigned to users") + # Check if role is assigned to any users via UserMandateRole + roleAssignments = self.db.getRecordset(UserMandateRole, recordFilter={"roleId": roleId}) + if roleAssignments: + raise ValueError(f"Cannot delete role '{role.roleLabel}' - it is assigned to users") # Check if role is used in any access rules - accessRules = self.getAccessRules(roleLabel=role.roleLabel) + accessRules = self.getAccessRules(roleId=roleId) if accessRules: raise ValueError(f"Cannot delete role '{role.roleLabel}' - it is used in access rules") @@ -2207,20 +2560,34 @@ class AppObjects: # Public Methods -def getInterface(currentUser: User) -> AppObjects: +def getInterface(currentUser: User, mandateId: Optional[str] = None) -> AppObjects: """ Returns a AppObjects instance for the current user. Handles initialization of database and records. + + Multi-Tenant Design: + - mandateId wird explizit übergeben (aus Request-Context / X-Mandate-Id Header) + + Args: + currentUser: User object + mandateId: Explicit mandate context (from request header). Required for non-sysadmin. + + Returns: + AppObjects instance for the user context """ if not currentUser: raise ValueError("Invalid user context: user is required") - # Create context key - contextKey = f"{currentUser.mandateId}_{currentUser.id}" + effectiveMandateId = mandateId + + # Create context key (user + mandate combination) + contextKey = f"{effectiveMandateId}_{currentUser.id}" # Create new instance if not exists if contextKey not in _gatewayInterfaces: - _gatewayInterfaces[contextKey] = AppObjects(currentUser) + instance = AppObjects(currentUser) + instance.setUserContext(currentUser, mandateId=effectiveMandateId) + _gatewayInterfaces[contextKey] = instance return _gatewayInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatObjects.py index 4c15ba89..10b202da 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatObjects.py @@ -178,12 +178,18 @@ class ChatObjects: Uses the JSON connector for data access with added language support. """ - def __init__(self, currentUser: Optional[User] = None): - """Initializes the Chat Interface.""" + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + """Initializes the Chat Interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ # Initialize variables self.currentUser = currentUser # Store User object directly self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId self.rbac = None # RBAC interface # Initialize services @@ -194,7 +200,7 @@ class ChatObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser) + self.setUserContext(currentUser, mandateId=mandateId) # ===== Generic Utility Methods ===== @@ -257,14 +263,24 @@ class ChatObjects: def _initializeServices(self): pass - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ self.currentUser = currentUser # Store User object directly self.userId = currentUser.id - self.mandateId = currentUser.mandateId + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId can be None for sysadmins performing cross-mandate operations + if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): + raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") # Add language settings self.userLanguage = currentUser.language # Default user language @@ -293,11 +309,11 @@ class ChatObjects: """Initializes the database connection directly.""" try: # Get configuration values with defaults - dbHost = APP_CONFIG.get("DB_CHAT_HOST", "_no_config_default_data") - dbDatabase = APP_CONFIG.get("DB_CHAT_DATABASE", "chat") - dbUser = APP_CONFIG.get("DB_CHAT_USER") - dbPassword = APP_CONFIG.get("DB_CHAT_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_CHAT_PORT", 5432)) + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_chat" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) # Create database connector directly self.db = DatabaseConnector( @@ -654,7 +670,7 @@ class ChatObjects: logs=logs, messages=messages, stats=stats, - mandateId=workflow.get("mandateId", self.currentUser.mandateId) + mandateId=workflow.get("mandateId", self.mandateId) ) except Exception as e: logger.error(f"Error validating workflow data: {str(e)}") @@ -695,7 +711,7 @@ class ChatObjects: logs=[], messages=[], stats=[], - mandateId=created.get("mandateId", self.currentUser.mandateId), + mandateId=created.get("mandateId", self.mandateId), workflowMode=created["workflowMode"], maxSteps=created.get("maxSteps", 1) ) @@ -1088,7 +1104,7 @@ class ChatObjects: logger.error(f"Error creating workflow message: {str(e)}") return None - def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Dict[str, Any]: + def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatMessage]: """Updates a workflow message if user has access to the workflow.""" try: @@ -1174,8 +1190,10 @@ class ChatObjects: logger.error(f"Error updating message documents: {str(e)}") if not updatedMessage: logger.warning(f"Failed to update message {messageId}") - - return updatedMessage + return None + + # Convert to ChatMessage model + return ChatMessage(**updatedMessage) except Exception as e: logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) raise ValueError(f"Error updating message {messageId}: {str(e)}") @@ -1716,7 +1734,7 @@ class ChatObjects: totalPages=totalPages ) - def getAutomationDefinition(self, automationId: str) -> Optional[Dict[str, Any]]: + def getAutomationDefinition(self, automationId: str) -> Optional[AutomationDefinition]: """Returns an automation definition by ID if user has access, with computed status.""" try: # Use RBAC filtering @@ -1736,12 +1754,14 @@ class ChatObjects: automation["executionLogs"] = [] # Enrich with user and mandate names self._enrichAutomationWithUserAndMandate(automation) - return automation + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in automation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error getting automation definition: {str(e)}") return None - def createAutomationDefinition(self, automationData: Dict[str, Any]) -> Dict[str, Any]: + def createAutomationDefinition(self, automationData: Dict[str, Any]) -> AutomationDefinition: """Creates a new automation definition, then triggers sync.""" try: # Ensure ID is present @@ -1777,12 +1797,14 @@ class ChatObjects: # Trigger automation change callback (async, don't wait) asyncio.create_task(self._notifyAutomationChanged()) - return createdAutomation + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in createdAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error creating automation definition: {str(e)}") raise - def updateAutomationDefinition(self, automationId: str, automationData: Dict[str, Any]) -> Dict[str, Any]: + def updateAutomationDefinition(self, automationId: str, automationData: Dict[str, Any]) -> AutomationDefinition: """Updates an automation definition, then triggers sync.""" try: # Check access @@ -1808,7 +1830,9 @@ class ChatObjects: # Trigger automation change callback (async, don't wait) asyncio.create_task(self._notifyAutomationChanged()) - return updatedAutomation + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in updatedAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error updating automation definition: {str(e)}") raise @@ -1870,19 +1894,28 @@ class ChatObjects: logger.error(f"Error notifying automation change: {str(e)}") -def getInterface(currentUser: Optional[User] = None) -> 'ChatObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ChatObjects': """ Returns a ChatObjects instance for the current user. Handles initialization of database and records. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. """ if not currentUser: raise ValueError("Invalid user context: user is required") + effectiveMandateId = str(mandateId) if mandateId else None + # Create context key - contextKey = f"{currentUser.mandateId}_{currentUser.id}" + contextKey = f"{effectiveMandateId}_{currentUser.id}" # Create new instance if not exists if contextKey not in _chatInterfaces: - _chatInterfaces[contextKey] = ChatObjects(currentUser) + _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId) + else: + # Update user context if needed + _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId) return _chatInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbComponentObjects.py index fee78fef..07cf235c 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbComponentObjects.py @@ -76,14 +76,21 @@ class ComponentObjects: # Initialize standard records if needed self._initRecords() - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ if not currentUser: logger.info("Initializing interface without user context") return self.currentUser = currentUser # Store User object directly self.userId = currentUser.id + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -116,11 +123,11 @@ class ComponentObjects: """Initializes the database connection directly.""" try: # Get configuration values with defaults - dbHost = APP_CONFIG.get("DB_MANAGEMENT_HOST", "_no_config_default_data") - dbDatabase = APP_CONFIG.get("DB_MANAGEMENT_DATABASE", "management") - dbUser = APP_CONFIG.get("DB_MANAGEMENT_USER") - dbPassword = APP_CONFIG.get("DB_MANAGEMENT_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_MANAGEMENT_PORT")) + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_management" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) # Create database connector directly self.db = DatabaseConnector( @@ -979,8 +986,8 @@ class ComponentObjects: fileSize = len(content) fileHash = hashlib.sha256(content).hexdigest() - # Ensure mandateId is valid - mandateId = self.currentUser.mandateId or "default" + # Use mandateId from context + mandateId = self.mandateId # Create FileItem instance fileItem = FileItem( @@ -1320,9 +1327,9 @@ class ComponentObjects: if "userId" not in settingsData: settingsData["userId"] = self.userId - # Ensure mandateId is set + # Ensure mandateId is set from context if "mandateId" not in settingsData: - settingsData["mandateId"] = self.currentUser.mandateId if self.currentUser else "default" + settingsData["mandateId"] = self.mandateId # Check if settings already exist for this user existingSettings = self.getVoiceSettings(settingsData["userId"]) @@ -1406,7 +1413,7 @@ class ComponentObjects: # Create default settings defaultSettings = { "userId": targetUserId, - "mandateId": self.currentUser.mandateId if self.currentUser else "default", + "mandateId": self.mandateId, "sttLanguage": "de-DE", "ttsLanguage": "de-DE", "ttsVoice": "de-DE-KatjaNeural", @@ -1494,9 +1501,9 @@ class ComponentObjects: if not all(c.isalpha() or c == "_" for c in subscriptionId): raise ValueError("subscriptionId must contain only letters and underscores") - # Set mandateId if not provided + # Set mandateId from context if "mandateId" not in subscriptionData: - subscriptionData["mandateId"] = self.currentUser.mandateId if self.currentUser else "default" + subscriptionData["mandateId"] = self.mandateId createdRecord = self.db.recordCreate(MessagingSubscription, subscriptionData) if not createdRecord or not createdRecord.get("id"): @@ -1741,12 +1748,18 @@ class ComponentObjects: return MessagingDelivery(**filteredDeliveries[0]) if filteredDeliveries else None -def getInterface(currentUser: Optional[User] = None) -> 'ComponentObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ComponentObjects': """ Returns a ComponentObjects instance. If currentUser is provided, initializes with user context. Otherwise, returns an instance with only database access. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. """ + effectiveMandateId = str(mandateId) if mandateId else None + # Create new instance if not exists if "default" not in _instancesManagement: _instancesManagement["default"] = ComponentObjects() @@ -1754,7 +1767,7 @@ def getInterface(currentUser: Optional[User] = None) -> 'ComponentObjects': interface = _instancesManagement["default"] if currentUser: - interface.setUserContext(currentUser) + interface.setUserContext(currentUser, mandateId=effectiveMandateId) else: logger.info("Returning interface without user context") diff --git a/modules/interfaces/interfaceDbRealEstateObjects.py b/modules/interfaces/interfaceDbRealEstateObjects.py index df0ac295..d475b8db 100644 --- a/modules/interfaces/interfaceDbRealEstateObjects.py +++ b/modules/interfaces/interfaceDbRealEstateObjects.py @@ -39,11 +39,17 @@ class RealEstateObjects: Handles CRUD operations on Real Estate entities. """ - def __init__(self, currentUser: Optional[User] = None): - """Initializes the Real Estate Interface.""" + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + """Initializes the Real Estate Interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId self.rbac = None # RBAC interface # Initialize database @@ -51,17 +57,17 @@ class RealEstateObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser) + self.setUserContext(currentUser, mandateId=mandateId) def _initializeDatabase(self): """Initialize PostgreSQL database connection.""" try: # Get database configuration from environment - dbHost = APP_CONFIG.get("DB_REALESTATE_HOST", "localhost") - dbDatabase = APP_CONFIG.get("DB_REALESTATE_DATABASE", "poweron_realestate") - dbUser = APP_CONFIG.get("DB_REALESTATE_USER") - dbPassword = APP_CONFIG.get("DB_REALESTATE_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_REALESTATE_PORT", 5432)) + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_realestate" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) # Initialize database connector self.db = DatabaseConnector( @@ -101,14 +107,24 @@ class RealEstateObjects: logger.warning(f"Error ensuring supporting tables exist: {e}") # Don't raise - tables will be created on-demand anyway - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ self.currentUser = currentUser self.userId = currentUser.id - self.mandateId = currentUser.mandateId + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId can be None for sysadmins performing cross-mandate operations + if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): + raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") # Initialize RBAC interface if not self.currentUser: @@ -239,14 +255,8 @@ class RealEstateObjects: def getParzellen(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Parzelle]: """Get all plots matching the filter.""" - original_gemeinde_value = None - # Resolve location names to IDs if needed if recordFilter: - # Save original value before resolution for fallback search - if "kontextGemeinde" in recordFilter: - original_gemeinde_value = recordFilter["kontextGemeinde"] - recordFilter = self._resolveLocationFilters(recordFilter) records = getRecordsetWithRBAC( @@ -256,23 +266,6 @@ class RealEstateObjects: recordFilter=recordFilter or {} ) - # Fallback: If no records found and we resolved a Gemeinde name, - # try searching with the original name for backwards compatibility - # (handles case where data has string names instead of UUIDs) - if not records and original_gemeinde_value and recordFilter and "kontextGemeinde" in recordFilter: - if recordFilter["kontextGemeinde"] != original_gemeinde_value: - logger.info(f"No results with resolved UUID, trying with original name '{original_gemeinde_value}'") - fallback_filter = recordFilter.copy() - fallback_filter["kontextGemeinde"] = original_gemeinde_value - records = getRecordsetWithRBAC( - self.db, - Parzelle, - self.currentUser, - recordFilter=fallback_filter - ) - if records: - logger.info(f"Found {len(records)} records using original name (legacy data format)") - return [Parzelle(**r) for r in records] def _resolveLocationFilters(self, recordFilter: Dict[str, Any]) -> Dict[str, Any]: @@ -799,15 +792,24 @@ class RealEstateObjects: raise -def getInterface(currentUser: User) -> RealEstateObjects: +def getInterface(currentUser: User, mandateId: Optional[str] = None) -> RealEstateObjects: """ Factory function to get or create a Real Estate interface instance for a user. Uses singleton pattern per user. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. """ - userKey = f"{currentUser.id}_{currentUser.mandateId}" + effectiveMandateId = str(mandateId) if mandateId else None + + userKey = f"{currentUser.id}_{effectiveMandateId}" if userKey not in _realEstateInterfaces: - _realEstateInterfaces[userKey] = RealEstateObjects(currentUser) + _realEstateInterfaces[userKey] = RealEstateObjects(currentUser, mandateId=effectiveMandateId) + else: + # Update user context if needed + _realEstateInterfaces[userKey].setUserContext(currentUser, mandateId=effectiveMandateId) return _realEstateInterfaces[userKey] diff --git a/modules/interfaces/interfaceDbTrusteeObjects.py b/modules/interfaces/interfaceDbTrusteeObjects.py index 54fc25c7..edb085fa 100644 --- a/modules/interfaces/interfaceDbTrusteeObjects.py +++ b/modules/interfaces/interfaceDbTrusteeObjects.py @@ -7,7 +7,8 @@ Manages trustee organisations, roles, access, contracts, documents, and position import logging import math -from typing import Dict, Any, List, Optional +import uuid +from typing import Dict, Any, List, Optional, Union from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG @@ -32,20 +33,27 @@ logger = logging.getLogger(__name__) _trusteeInterfaces = {} -def getInterface(currentUser: User) -> "TrusteeObjects": - """Get or create a TrusteeObjects instance for the given user context.""" +def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = None) -> "TrusteeObjects": + """Get or create a TrusteeObjects instance for the given user context. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + """ global _trusteeInterfaces if not currentUser or not currentUser.id: raise ValueError("Valid user context required") - cacheKey = f"{currentUser.id}_{currentUser.mandateId}" + effectiveMandateId = str(mandateId) if mandateId else None + + cacheKey = f"{currentUser.id}_{effectiveMandateId}" if cacheKey not in _trusteeInterfaces: - _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser) + _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser, mandateId=effectiveMandateId) else: # Update user context if needed - _trusteeInterfaces[cacheKey].setUserContext(currentUser) + _trusteeInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId) return _trusteeInterfaces[cacheKey] @@ -56,11 +64,17 @@ class TrusteeObjects: Manages trustee organisations, roles, access, contracts, documents, and positions. """ - def __init__(self, currentUser: Optional[User] = None): - """Initializes the Trustee Interface.""" + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + """Initializes the Trustee Interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId self.rbac = None # Initialize database @@ -68,20 +82,30 @@ class TrusteeObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser) + self.setUserContext(currentUser, mandateId=mandateId) - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ if not currentUser: logger.info("Initializing interface without user context") return self.currentUser = currentUser self.userId = currentUser.id - self.mandateId = currentUser.mandateId + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId can be None for sysadmins performing cross-mandate operations + if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): + raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") self.userLanguage = currentUser.language @@ -104,11 +128,11 @@ class TrusteeObjects: def _initializeDatabase(self): """Initializes the database connection directly.""" try: - dbHost = APP_CONFIG.get("DB_TRUSTEE_HOST", APP_CONFIG.get("DB_CHAT_HOST", "_no_config_default_data")) - dbDatabase = APP_CONFIG.get("DB_TRUSTEE_DATABASE", "trustee") - dbUser = APP_CONFIG.get("DB_TRUSTEE_USER", APP_CONFIG.get("DB_CHAT_USER")) - dbPassword = APP_CONFIG.get("DB_TRUSTEE_PASSWORD_SECRET", APP_CONFIG.get("DB_CHAT_PASSWORD_SECRET")) - dbPort = int(APP_CONFIG.get("DB_TRUSTEE_PORT", APP_CONFIG.get("DB_CHAT_PORT", 5432))) + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_trustee" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) self.db = DatabaseConnector( dbHost=dbHost, @@ -174,7 +198,7 @@ class TrusteeObjects: # ===== Organisation CRUD ===== - def createOrganisation(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createOrganisation(self, data: Dict[str, Any]) -> Optional[TrusteeOrganisation]: """Create a new organisation.""" if not self.checkRbacPermission(TrusteeOrganisation, "create"): logger.warning(f"User {self.userId} lacks permission to create organisation") @@ -196,13 +220,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeOrganisation, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteeOrganisation(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getOrganisation(self, orgId: str) -> Optional[Dict[str, Any]]: + def getOrganisation(self, orgId: str) -> Optional[TrusteeOrganisation]: """Get a single organisation by ID.""" records = self.db.getRecordset(TrusteeOrganisation, recordFilter={"id": orgId}) - return records[0] if records else None + if not records: + return None + return TrusteeOrganisation(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllOrganisations(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all organisations with RBAC filtering. @@ -214,7 +240,7 @@ class TrusteeObjects: - New organisations wouldn't be visible without an access record """ # Debug: Log user info and permissions - logger.debug(f"getAllOrganisations called for user {self.userId}, roles: {self.currentUser.roleLabels if self.currentUser else 'None'}, mandateId: {self.mandateId}") + logger.debug(f"getAllOrganisations called for user {self.userId}, mandateId: {self.mandateId}") # System RBAC filtering (filters by mandate for GROUP access level) records = getRecordsetWithRBAC( @@ -247,7 +273,7 @@ class TrusteeObjects: totalPages=totalPages ) - def updateOrganisation(self, orgId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updateOrganisation(self, orgId: str, data: Dict[str, Any]) -> Optional[TrusteeOrganisation]: """Update an organisation.""" if not self.checkRbacPermission(TrusteeOrganisation, "update"): logger.warning(f"User {self.userId} lacks permission to update organisation") @@ -260,7 +286,9 @@ class TrusteeObjects: data["id"] = orgId updatedRecord = self.db.recordModify(TrusteeOrganisation, orgId, data) - return updatedRecord + if not updatedRecord: + return None + return TrusteeOrganisation(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deleteOrganisation(self, orgId: str) -> bool: """Delete an organisation.""" @@ -272,7 +300,7 @@ class TrusteeObjects: # ===== Role CRUD ===== - def createRole(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createRole(self, data: Dict[str, Any]) -> Optional[TrusteeRole]: """Create a new role (sysadmin only).""" if not self.checkRbacPermission(TrusteeRole, "create"): logger.warning(f"User {self.userId} lacks permission to create role") @@ -287,13 +315,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeRole, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteeRole(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getRole(self, roleId: str) -> Optional[Dict[str, Any]]: + def getRole(self, roleId: str) -> Optional[TrusteeRole]: """Get a single role by ID.""" records = self.db.getRecordset(TrusteeRole, recordFilter={"id": roleId}) - return records[0] if records else None + if not records: + return None + return TrusteeRole(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllRoles(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all roles with RBAC filtering. @@ -338,7 +368,7 @@ class TrusteeObjects: totalPages=totalPages ) - def updateRole(self, roleId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updateRole(self, roleId: str, data: Dict[str, Any]) -> Optional[TrusteeRole]: """Update a role (sysadmin only).""" if not self.checkRbacPermission(TrusteeRole, "update"): logger.warning(f"User {self.userId} lacks permission to update role") @@ -346,7 +376,9 @@ class TrusteeObjects: data["id"] = roleId updatedRecord = self.db.recordModify(TrusteeRole, roleId, data) - return updatedRecord + if not updatedRecord: + return None + return TrusteeRole(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deleteRole(self, roleId: str) -> bool: """Delete a role (sysadmin only, not if in use).""" @@ -364,7 +396,7 @@ class TrusteeObjects: # ===== Access CRUD ===== - def createAccess(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createAccess(self, data: Dict[str, Any]) -> Optional[TrusteeAccess]: """Create a new access record. Requires admin role for the organisation or ALL access level.""" # Check system RBAC first if not self.checkRbacPermission(TrusteeAccess, "create"): @@ -389,13 +421,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeAccess, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteeAccess(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getAccess(self, accessId: str) -> Optional[Dict[str, Any]]: + def getAccess(self, accessId: str) -> Optional[TrusteeAccess]: """Get a single access record by ID.""" records = self.db.getRecordset(TrusteeAccess, recordFilter={"id": accessId}) - return records[0] if records else None + if not records: + return None + return TrusteeAccess(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllAccess(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all access records with RBAC filtering + feature-level filtering. @@ -451,7 +485,7 @@ class TrusteeObjects: totalPages=totalPages ) - def getAccessByOrganisation(self, organisationId: str) -> List[Dict[str, Any]]: + def getAccessByOrganisation(self, organisationId: str) -> List[TrusteeAccess]: """Get all access records for a specific organisation. Requires admin role for the organisation. @@ -461,15 +495,16 @@ class TrusteeObjects: logger.warning(f"User {self.userId} lacks admin role for organisation {organisationId}") return [] - return getRecordsetWithRBAC( + records = getRecordsetWithRBAC( connector=self.db, modelClass=TrusteeAccess, currentUser=self.currentUser, recordFilter={"organisationId": organisationId}, orderBy="id" ) + return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] - def getAccessByUser(self, userId: str) -> List[Dict[str, Any]]: + def getAccessByUser(self, userId: str) -> List[TrusteeAccess]: """Get all access records for a specific user. Users with ALL access level see all access records. @@ -486,7 +521,7 @@ class TrusteeObjects: # Users with ALL access level (from system RBAC) see all records accessLevel = self.getRbacAccessLevel(TrusteeAccess, "read") if accessLevel == AccessLevel.ALL: - return records + return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] # Filter to only organisations where current user has admin role userAccess = self.getAllUserAccess(self.userId) @@ -495,9 +530,10 @@ class TrusteeObjects: if access.get("roleId") == "admin": adminOrgs.add(access.get("organisationId")) - return [r for r in records if r.get("organisationId") in adminOrgs] + filtered = [r for r in records if r.get("organisationId") in adminOrgs] + return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] - def updateAccess(self, accessId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updateAccess(self, accessId: str, data: Dict[str, Any]) -> Optional[TrusteeAccess]: """Update an access record. Requires admin role for the organisation or ALL access level.""" # Check system RBAC first if not self.checkRbacPermission(TrusteeAccess, "update"): @@ -524,7 +560,9 @@ class TrusteeObjects: data["id"] = accessId updatedRecord = self.db.recordModify(TrusteeAccess, accessId, data) - return updatedRecord + if not updatedRecord: + return None + return TrusteeAccess(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deleteAccess(self, accessId: str) -> bool: """Delete an access record. Requires admin role for the organisation or ALL access level.""" @@ -555,7 +593,7 @@ class TrusteeObjects: # ===== Contract CRUD ===== - def createContract(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createContract(self, data: Dict[str, Any]) -> Optional[TrusteeContract]: """Create a new contract.""" organisationId = data.get("organisationId") @@ -572,13 +610,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeContract, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteeContract(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getContract(self, contractId: str) -> Optional[Dict[str, Any]]: + def getContract(self, contractId: str) -> Optional[TrusteeContract]: """Get a single contract by ID.""" records = self.db.getRecordset(TrusteeContract, recordFilter={"id": contractId}) - return records[0] if records else None + if not records: + return None + return TrusteeContract(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllContracts(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all contracts with RBAC filtering + feature-level access filtering.""" @@ -614,7 +654,7 @@ class TrusteeObjects: totalPages=totalPages ) - def getContractsByOrganisation(self, organisationId: str) -> List[Dict[str, Any]]: + def getContractsByOrganisation(self, organisationId: str) -> List[TrusteeContract]: """Get all contracts for a specific organisation.""" # Step 1: System RBAC filtering records = getRecordsetWithRBAC( @@ -626,9 +666,10 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - return self.filterRecordsByTrusteeAccess(records, TrusteeContract) + filtered = self.filterRecordsByTrusteeAccess(records, TrusteeContract) + return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] - def updateContract(self, contractId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updateContract(self, contractId: str, data: Dict[str, Any]) -> Optional[TrusteeContract]: """Update a contract (organisationId is immutable).""" # Get existing contract to check organisation existingRecords = self.db.getRecordset(TrusteeContract, recordFilter={"id": contractId}) @@ -652,7 +693,9 @@ class TrusteeObjects: data["id"] = contractId updatedRecord = self.db.recordModify(TrusteeContract, contractId, data) - return updatedRecord + if not updatedRecord: + return None + return TrusteeContract(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deleteContract(self, contractId: str) -> bool: """Delete a contract.""" @@ -675,7 +718,7 @@ class TrusteeObjects: # ===== Document CRUD ===== - def createDocument(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createDocument(self, data: Dict[str, Any]) -> Optional[TrusteeDocument]: """Create a new document.""" organisationId = data.get("organisationId") contractId = data.get("contractId") @@ -693,20 +736,19 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeDocument, data) if createdRecord and createdRecord.get("id"): - # Remove binary data from response - createdRecord.pop("documentData", None) - return createdRecord + # Remove binary data and metadata from Pydantic model + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_") and k != "documentData"} + return TrusteeDocument(**cleanedRecord) return None - def getDocument(self, documentId: str) -> Optional[Dict[str, Any]]: + def getDocument(self, documentId: str) -> Optional[TrusteeDocument]: """Get a single document by ID (metadata only).""" records = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) - if records: - record = records[0] - # Remove binary data from response - record.pop("documentData", None) - return record - return None + if not records: + return None + # Remove binary data and metadata from Pydantic model + cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_") and k != "documentData"} + return TrusteeDocument(**cleanedRecord) def getDocumentData(self, documentId: str) -> Optional[bytes]: """Get document binary data.""" @@ -755,7 +797,7 @@ class TrusteeObjects: totalPages=totalPages ) - def getDocumentsByContract(self, contractId: str) -> List[Dict[str, Any]]: + def getDocumentsByContract(self, contractId: str) -> List[TrusteeDocument]: """Get all documents for a specific contract.""" # Step 1: System RBAC filtering records = getRecordsetWithRBAC( @@ -767,13 +809,15 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - records = self.filterRecordsByTrusteeAccess(records, TrusteeDocument) + filtered = self.filterRecordsByTrusteeAccess(records, TrusteeDocument) - for record in records: - record.pop("documentData", None) - return records + result = [] + for record in filtered: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} + result.append(TrusteeDocument(**cleanedRecord)) + return result - def updateDocument(self, documentId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updateDocument(self, documentId: str, data: Dict[str, Any]) -> Optional[TrusteeDocument]: """Update a document.""" # Get existing document to check organisation and creator existingRecords = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) @@ -795,9 +839,10 @@ class TrusteeObjects: data["id"] = documentId updatedRecord = self.db.recordModify(TrusteeDocument, documentId, data) - if updatedRecord: - updatedRecord.pop("documentData", None) - return updatedRecord + if not updatedRecord: + return None + cleanedRecord = {k: v for k, v in updatedRecord.items() if not k.startswith("_") and k != "documentData"} + return TrusteeDocument(**cleanedRecord) def deleteDocument(self, documentId: str) -> bool: """Delete a document.""" @@ -823,7 +868,7 @@ class TrusteeObjects: # ===== Position CRUD ===== - def createPosition(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createPosition(self, data: Dict[str, Any]) -> Optional[TrusteePosition]: """Create a new position.""" organisationId = data.get("organisationId") contractId = data.get("contractId") @@ -847,13 +892,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteePosition, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteePosition(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getPosition(self, positionId: str) -> Optional[Dict[str, Any]]: + def getPosition(self, positionId: str) -> Optional[TrusteePosition]: """Get a single position by ID.""" records = self.db.getRecordset(TrusteePosition, recordFilter={"id": positionId}) - return records[0] if records else None + if not records: + return None + return TrusteePosition(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllPositions(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all positions with RBAC filtering + feature-level access filtering.""" @@ -890,7 +937,7 @@ class TrusteeObjects: totalPages=totalPages ) - def getPositionsByContract(self, contractId: str) -> List[Dict[str, Any]]: + def getPositionsByContract(self, contractId: str) -> List[TrusteePosition]: """Get all positions for a specific contract.""" # Step 1: System RBAC filtering records = getRecordsetWithRBAC( @@ -902,9 +949,10 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - return self.filterRecordsByTrusteeAccess(records, TrusteePosition) + filtered = self.filterRecordsByTrusteeAccess(records, TrusteePosition) + return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] - def getPositionsByOrganisation(self, organisationId: str) -> List[Dict[str, Any]]: + def getPositionsByOrganisation(self, organisationId: str) -> List[TrusteePosition]: """Get all positions for a specific organisation.""" # Step 1: System RBAC filtering records = getRecordsetWithRBAC( @@ -916,9 +964,10 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - return self.filterRecordsByTrusteeAccess(records, TrusteePosition) + filtered = self.filterRecordsByTrusteeAccess(records, TrusteePosition) + return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] - def updatePosition(self, positionId: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def updatePosition(self, positionId: str, data: Dict[str, Any]) -> Optional[TrusteePosition]: """Update a position.""" # Get existing position to check organisation and creator existingRecords = self.db.getRecordset(TrusteePosition, recordFilter={"id": positionId}) @@ -940,7 +989,9 @@ class TrusteeObjects: data["id"] = positionId updatedRecord = self.db.recordModify(TrusteePosition, positionId, data) - return updatedRecord + if not updatedRecord: + return None + return TrusteePosition(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deletePosition(self, positionId: str) -> bool: """Delete a position.""" @@ -966,7 +1017,7 @@ class TrusteeObjects: # ===== Position-Document Link CRUD ===== - def createPositionDocument(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + def createPositionDocument(self, data: Dict[str, Any]) -> Optional[TrusteePositionDocument]: """Create a new position-document link.""" organisationId = data.get("organisationId") contractId = data.get("contractId") @@ -984,13 +1035,15 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteePositionDocument, data) if createdRecord and createdRecord.get("id"): - return createdRecord + return TrusteePositionDocument(**{k: v for k, v in createdRecord.items() if not k.startswith("_")}) return None - def getPositionDocument(self, linkId: str) -> Optional[Dict[str, Any]]: + def getPositionDocument(self, linkId: str) -> Optional[TrusteePositionDocument]: """Get a single position-document link by ID.""" records = self.db.getRecordset(TrusteePositionDocument, recordFilter={"id": linkId}) - return records[0] if records else None + if not records: + return None + return TrusteePositionDocument(**{k: v for k, v in records[0].items() if not k.startswith("_")}) def getAllPositionDocuments(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all position-document links with RBAC filtering + feature-level access filtering.""" @@ -1027,7 +1080,7 @@ class TrusteeObjects: totalPages=totalPages ) - def getDocumentsForPosition(self, positionId: str) -> List[Dict[str, Any]]: + def getDocumentsForPosition(self, positionId: str) -> List[TrusteePositionDocument]: """Get all documents linked to a position.""" # Step 1: System RBAC filtering links = getRecordsetWithRBAC( @@ -1039,9 +1092,10 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - return self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) + filtered = self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) + return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] - def getPositionsForDocument(self, documentId: str) -> List[Dict[str, Any]]: + def getPositionsForDocument(self, documentId: str) -> List[TrusteePositionDocument]: """Get all positions linked to a document.""" # Step 1: System RBAC filtering links = getRecordsetWithRBAC( @@ -1053,7 +1107,8 @@ class TrusteeObjects: ) # Step 2: Feature-level filtering based on trustee.access - return self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) + filtered = self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) + return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] def deletePositionDocument(self, linkId: str) -> bool: """Delete a position-document link.""" diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py new file mode 100644 index 00000000..697658b3 --- /dev/null +++ b/modules/interfaces/interfaceFeatures.py @@ -0,0 +1,478 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Feature Instance Management Interface. + +Multi-Tenant Design: +- Feature-Instanzen gehören zu Mandanten +- Template-Rollen werden bei Erstellung kopiert +- Synchronisation von Templates ist explizit (nicht automatisch) +""" + +import logging +import uuid +from typing import List, Dict, Any, Optional + +from modules.datamodels.datamodelFeatures import Feature, FeatureInstance +from modules.datamodels.datamodelRbac import Role, AccessRule +from modules.connectors.connectorDbPostgre import DatabaseConnector + +logger = logging.getLogger(__name__) + + +class FeatureInterface: + """ + Interface for Feature and FeatureInstance management. + + Responsibilities: + - CRUD operations for Features and FeatureInstances + - Template role copying on instance creation + - Template synchronization for existing instances + """ + + def __init__(self, db: DatabaseConnector): + """ + Initialize Feature interface. + + Args: + db: DatabaseConnector instance (DbApp database) + """ + self.db = db + + # ============================================ + # Feature Methods (Global Feature Definitions) + # ============================================ + + def getFeature(self, featureCode: str) -> Optional[Feature]: + """ + Get a feature by code. + + Args: + featureCode: Feature code (e.g., "trustee", "chatbot") + + Returns: + Feature object or None + """ + try: + records = self.db.getRecordset(Feature, recordFilter={"code": featureCode}) + if not records: + return None + cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + return Feature(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting feature {featureCode}: {e}") + return None + + def getAllFeatures(self) -> List[Feature]: + """ + Get all available features. + + Returns: + List of Feature objects + """ + try: + records = self.db.getRecordset(Feature) + result = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + result.append(Feature(**cleanedRecord)) + return result + except Exception as e: + logger.error(f"Error getting all features: {e}") + return [] + + def createFeature(self, code: str, label: Dict[str, str], icon: str = "mdi-puzzle") -> Feature: + """ + Create a new feature definition. + + Args: + code: Unique feature code (e.g., "trustee") + label: I18n labels (e.g., {"en": "Trustee", "de": "Treuhand"}) + icon: Icon identifier + + Returns: + Created Feature object + """ + try: + feature = Feature(code=code, label=label, icon=icon) + createdRecord = self.db.recordCreate(Feature, feature.model_dump()) + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + return Feature(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating feature {code}: {e}") + raise ValueError(f"Failed to create feature: {e}") + + # ============================================ + # Feature Instance Methods + # ============================================ + + def getFeatureInstance(self, instanceId: str) -> Optional[FeatureInstance]: + """ + Get a feature instance by ID. + + Args: + instanceId: FeatureInstance ID + + Returns: + FeatureInstance object or None + """ + try: + records = self.db.getRecordset(FeatureInstance, recordFilter={"id": instanceId}) + if not records: + return None + cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + return FeatureInstance(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting feature instance {instanceId}: {e}") + return None + + def getFeatureInstancesForMandate(self, mandateId: str, featureCode: Optional[str] = None) -> List[FeatureInstance]: + """ + Get all feature instances for a mandate. + + Args: + mandateId: Mandate ID + featureCode: Optional filter by feature code + + Returns: + List of FeatureInstance objects + """ + try: + recordFilter = {"mandateId": mandateId} + if featureCode: + recordFilter["featureCode"] = featureCode + records = self.db.getRecordset(FeatureInstance, recordFilter=recordFilter) + result = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + result.append(FeatureInstance(**cleanedRecord)) + return result + except Exception as e: + logger.error(f"Error getting feature instances for mandate {mandateId}: {e}") + return [] + + def createFeatureInstance( + self, + featureCode: str, + mandateId: str, + label: str, + copyTemplateRoles: bool = True + ) -> FeatureInstance: + """ + Create a new feature instance for a mandate. + + Optionally copies global template roles for this feature. + + WICHTIG: Templates werden NUR bei Erstellung kopiert. + Spätere Template-Änderungen werden NICHT automatisch propagiert. + Für manuelle Nachsynchronisation siehe syncRolesFromTemplate(). + + Args: + featureCode: Feature code (e.g., "trustee") + mandateId: Mandate ID + label: Instance label (e.g., "Buchhaltung 2025") + copyTemplateRoles: Whether to copy template roles + + Returns: + Created FeatureInstance object + """ + try: + # Create instance + instance = FeatureInstance( + featureCode=featureCode, + mandateId=mandateId, + label=label, + enabled=True + ) + createdInstance = self.db.recordCreate(FeatureInstance, instance.model_dump()) + + if not createdInstance: + raise ValueError("Failed to create feature instance record") + + instanceId = createdInstance.get("id") + + # Copy template roles if requested + if copyTemplateRoles: + self._copyTemplateRoles(featureCode, mandateId, instanceId) + + cleanedRecord = {k: v for k, v in createdInstance.items() if not k.startswith("_")} + return FeatureInstance(**cleanedRecord) + + except Exception as e: + logger.error(f"Error creating feature instance: {e}") + raise ValueError(f"Failed to create feature instance: {e}") + + def _copyTemplateRoles(self, featureCode: str, mandateId: str, instanceId: str) -> int: + """ + Copy global template roles for a feature to a new instance. + + Args: + featureCode: Feature code + mandateId: Mandate ID + instanceId: New FeatureInstance ID + + Returns: + Number of roles copied + """ + try: + # Find global template roles for this feature (mandateId=None) + globalRoles = self.db.getRecordset( + Role, + recordFilter={"featureCode": featureCode, "mandateId": None} + ) + + if not globalRoles: + logger.debug(f"No template roles found for feature {featureCode}") + return 0 + + templateRoleIds = [r.get("id") for r in globalRoles] + + # BULK: Load all template AccessRules in one query + allTemplateRules = [] + for roleId in templateRoleIds: + rules = self.db.getRecordset(AccessRule, recordFilter={"roleId": roleId}) + allTemplateRules.extend([(roleId, r) for r in rules]) + + # Index for fast lookup: roleId -> rules + rulesByRoleId = {} + for roleId, rule in allTemplateRules: + if roleId not in rulesByRoleId: + rulesByRoleId[roleId] = [] + rulesByRoleId[roleId].append(rule) + + # Copy roles and their AccessRules + copiedCount = 0 + for templateRole in globalRoles: + newRoleId = str(uuid.uuid4()) + + # Create new role for this instance + newRole = Role( + id=newRoleId, + roleLabel=templateRole.get("roleLabel"), + description=templateRole.get("description", {}), + featureCode=featureCode, + mandateId=mandateId, + featureInstanceId=instanceId, + isSystemRole=False + ) + self.db.recordCreate(Role, newRole.model_dump()) + + # Copy AccessRules for this role + templateRulesForRole = rulesByRoleId.get(templateRole.get("id"), []) + for rule in templateRulesForRole: + newRule = AccessRule( + id=str(uuid.uuid4()), + roleId=newRoleId, + context=rule.get("context"), + item=rule.get("item"), + view=rule.get("view", False), + read=rule.get("read"), + create=rule.get("create"), + update=rule.get("update"), + delete=rule.get("delete") + ) + self.db.recordCreate(AccessRule, newRule.model_dump()) + + copiedCount += 1 + + logger.info(f"Copied {copiedCount} template roles for instance {instanceId}") + return copiedCount + + except Exception as e: + logger.error(f"Error copying template roles: {e}") + return 0 + + def syncRolesFromTemplate(self, featureInstanceId: str, addOnly: bool = True) -> Dict[str, int]: + """ + Synchronize roles of a feature instance with current templates. + + WICHTIG: Templates werden NUR bei Erstellung einer neuen FeatureInstance kopiert. + Diese Sync-Funktion ist für manuelle Nachsynchronisation gedacht, nicht für + automatische Propagation von Template-Änderungen. + + Args: + featureInstanceId: ID of the instance to sync + addOnly: If True, only add missing roles. If False, also remove extras. + + Returns: + Dict with added/removed/unchanged counts + """ + try: + instance = self.getFeatureInstance(featureInstanceId) + if not instance: + raise ValueError(f"FeatureInstance {featureInstanceId} not found") + + featureCode = instance.featureCode + mandateId = instance.mandateId + + # Get current template roles + templateRoles = self.db.getRecordset( + Role, + recordFilter={"featureCode": featureCode, "mandateId": None} + ) + templateLabels = {r.get("roleLabel") for r in templateRoles} + + # Get current instance roles + instanceRoles = self.db.getRecordset( + Role, + recordFilter={"featureInstanceId": featureInstanceId} + ) + instanceLabels = {r.get("roleLabel") for r in instanceRoles} + + result = {"added": 0, "removed": 0, "unchanged": 0} + + # Add missing roles + for templateRole in templateRoles: + if templateRole.get("roleLabel") not in instanceLabels: + # Copy this role + newRoleId = str(uuid.uuid4()) + newRole = Role( + id=newRoleId, + roleLabel=templateRole.get("roleLabel"), + description=templateRole.get("description", {}), + featureCode=featureCode, + mandateId=mandateId, + featureInstanceId=featureInstanceId, + isSystemRole=False + ) + self.db.recordCreate(Role, newRole.model_dump()) + + # Copy AccessRules + templateRules = self.db.getRecordset( + AccessRule, + recordFilter={"roleId": templateRole.get("id")} + ) + for rule in templateRules: + newRule = AccessRule( + id=str(uuid.uuid4()), + roleId=newRoleId, + context=rule.get("context"), + item=rule.get("item"), + view=rule.get("view", False), + read=rule.get("read"), + create=rule.get("create"), + update=rule.get("update"), + delete=rule.get("delete") + ) + self.db.recordCreate(AccessRule, newRule.model_dump()) + + result["added"] += 1 + else: + result["unchanged"] += 1 + + # Remove extra roles (optional) + if not addOnly: + from modules.datamodels.datamodelMembership import FeatureAccessRole + + for instanceRole in instanceRoles: + if instanceRole.get("roleLabel") not in templateLabels: + # Check if role is still in use + usages = self.db.getRecordset( + FeatureAccessRole, + recordFilter={"roleId": instanceRole.get("id")} + ) + if not usages: + self.db.recordDelete(Role, instanceRole.get("id")) + result["removed"] += 1 + + logger.info(f"Synced roles for instance {featureInstanceId}: {result}") + return result + + except Exception as e: + logger.error(f"Error syncing roles from template: {e}") + raise ValueError(f"Failed to sync roles: {e}") + + def deleteFeatureInstance(self, instanceId: str) -> bool: + """ + Delete a feature instance. + CASCADE will delete associated roles and access records. + + Args: + instanceId: FeatureInstance ID + + Returns: + True if deleted + """ + try: + instance = self.getFeatureInstance(instanceId) + if not instance: + return False + + return self.db.recordDelete(FeatureInstance, instanceId) + except Exception as e: + logger.error(f"Error deleting feature instance {instanceId}: {e}") + raise ValueError(f"Failed to delete feature instance: {e}") + + # ============================================ + # Template Role Methods (Global) + # ============================================ + + def getTemplateRoles(self, featureCode: Optional[str] = None) -> List[Role]: + """ + Get global template roles (mandateId=None). + + Args: + featureCode: Optional filter by feature code + + Returns: + List of Role objects + """ + try: + recordFilter = {"mandateId": None} + if featureCode: + recordFilter["featureCode"] = featureCode + records = self.db.getRecordset(Role, recordFilter=recordFilter) + result = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + result.append(Role(**cleanedRecord)) + return result + except Exception as e: + logger.error(f"Error getting template roles: {e}") + return [] + + def createTemplateRole( + self, + roleLabel: str, + featureCode: str, + description: Dict[str, str] = None + ) -> Role: + """ + Create a global template role for a feature. + + Args: + roleLabel: Role label (e.g., "admin", "viewer") + featureCode: Feature code this role belongs to + description: I18n descriptions + + Returns: + Created Role object + """ + try: + role = Role( + roleLabel=roleLabel, + description=description or {}, + featureCode=featureCode, + mandateId=None, # Global template + featureInstanceId=None, + isSystemRole=False + ) + createdRecord = self.db.recordCreate(Role, role.model_dump()) + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + return Role(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating template role: {e}") + raise ValueError(f"Failed to create template role: {e}") + + +def getFeatureInterface(db: DatabaseConnector) -> FeatureInterface: + """ + Factory function to get a FeatureInterface instance. + + Args: + db: DatabaseConnector instance (DbApp database) + + Returns: + FeatureInterface instance + """ + return FeatureInterface(db) diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index e232ae95..26515e94 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -3,6 +3,10 @@ """ RBAC helper functions for interfaces. Provides RBAC filtering for database queries without connectors importing security. + +Multi-Tenant Design: +- mandateId kommt aus Request-Context (X-Mandate-Id Header) +- GROUP-Filter verwendet expliziten mandateId Parameter """ import logging @@ -24,24 +28,33 @@ def getRecordsetWithRBAC( recordFilter: Dict[str, Any] = None, orderBy: str = None, limit: int = None, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, ) -> List[Dict[str, Any]]: """ Get records with RBAC filtering applied at database level. This function wraps connector.getRecordset() with RBAC logic. + Multi-Tenant Design: + - mandateId wird explizit übergeben (aus Request-Context / X-Mandate-Id Header) + Args: connector: DatabaseConnector instance modelClass: Pydantic model class for the table - currentUser: User object with roleLabels + currentUser: User object recordFilter: Additional record filters orderBy: Field to order by (defaults to "id") limit: Maximum number of records to return + mandateId: Explicit mandate context (from request header). Required for GROUP access. + featureInstanceId: Explicit feature instance context Returns: List of filtered records """ table = modelClass.__name__ + effectiveMandateId = mandateId + try: if not connector._ensureTableExists(modelClass): return [] @@ -53,7 +66,9 @@ def getRecordsetWithRBAC( permissions = rbacInstance.getUserPermissions( currentUser, AccessRuleContext.DATA, - table + table, + mandateId=effectiveMandateId, + featureInstanceId=featureInstanceId ) # Check view permission first @@ -66,7 +81,13 @@ def getRecordsetWithRBAC( whereValues = [] # Add RBAC WHERE clause based on read permission - rbacWhereClause = buildRbacWhereClause(permissions, currentUser, table, connector) + rbacWhereClause = buildRbacWhereClause( + permissions, + currentUser, + table, + connector, + mandateId=effectiveMandateId + ) if rbacWhereClause: whereConditions.append(rbacWhereClause["condition"]) whereValues.extend(rbacWhereClause["values"]) @@ -155,17 +176,21 @@ def buildRbacWhereClause( permissions: UserPermissions, currentUser: User, table: str, - connector # DatabaseConnector instance for connection access + connector, # DatabaseConnector instance for connection access + mandateId: Optional[str] = None ) -> Optional[Dict[str, Any]]: """ Build RBAC WHERE clause based on permissions and access level. - Moved from connector to interfaces. + + Multi-Tenant Design: + - mandateId wird explizit übergeben (aus Request-Context / X-Mandate-Id Header) Args: permissions: UserPermissions object currentUser: User object table: Table name connector: DatabaseConnector instance (needed for GROUP queries) + mandateId: Explicit mandate context (from request header). Required for GROUP access. Returns: Dictionary with "condition" and "values" keys, or None if no filtering needed @@ -201,7 +226,9 @@ def buildRbacWhereClause( # Group records - filter by mandateId if readLevel == AccessLevel.GROUP: - if not currentUser.mandateId: + effectiveMandateId = mandateId + + if not effectiveMandateId: logger.warning(f"User {currentUser.id} has no mandateId for GROUP access") return {"condition": "1 = 0", "values": []} @@ -209,7 +236,7 @@ def buildRbacWhereClause( if table == "UserInDB": return { "condition": '"mandateId" = %s', - "values": [currentUser.mandateId] + "values": [effectiveMandateId] } # For UserConnection, need to join with UserInDB or filter by mandateId in user elif table == "UserConnection": @@ -218,7 +245,7 @@ def buildRbacWhereClause( with connector.connection.cursor() as cursor: cursor.execute( 'SELECT "id" FROM "UserInDB" WHERE "mandateId" = %s', - (currentUser.mandateId,) + (effectiveMandateId,) ) users = cursor.fetchall() userIds = [u["id"] for u in users] @@ -236,8 +263,7 @@ def buildRbacWhereClause( else: return { "condition": '"mandateId" = %s', - "values": [currentUser.mandateId] + "values": [effectiveMandateId] } return None - diff --git a/modules/interfaces/interfaceVoiceObjects.py b/modules/interfaces/interfaceVoiceObjects.py index ee86910f..0c28f81d 100644 --- a/modules/interfaces/interfaceVoiceObjects.py +++ b/modules/interfaces/interfaceVoiceObjects.py @@ -31,19 +31,26 @@ class VoiceObjects: self.userId: Optional[str] = None self._google_speech_connector: Optional[ConnectorGoogleSpeech] = None - def setUserContext(self, currentUser: User): - """Set the user context for the interface.""" + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + """Set the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + """ if not currentUser: logger.info("Initializing voice interface without user context") return self.currentUser = currentUser self.userId = currentUser.id + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId if not self.userId: raise ValueError("Invalid user context: id is required") - logger.debug(f"Voice interface user context set: userId={self.userId}") + logger.debug(f"Voice interface user context set: userId={self.userId}, mandateId={self.mandateId}") def _getGoogleSpeechConnector(self) -> ConnectorGoogleSpeech: """Get or create Google Cloud Speech connector instance.""" @@ -308,11 +315,11 @@ class VoiceObjects: try: logger.info(f"Creating voice settings: {settingsData}") - # Ensure mandateId is set from user context if not provided + # Ensure mandateId is set from context if not provided if "mandateId" not in settingsData or not settingsData["mandateId"]: - if not self.currentUser or not self.currentUser.mandateId: - raise ValueError("mandateId is required but not provided and user context has no mandateId") - settingsData["mandateId"] = self.currentUser.mandateId + if not self.mandateId: + raise ValueError("mandateId is required but not provided and context has no mandateId") + settingsData["mandateId"] = self.mandateId # Add timestamps currentTime = getUtcTimestamp() @@ -376,7 +383,7 @@ class VoiceObjects: # Create default settings if none exist defaultSettings = { "userId": userId, - "mandateId": self.currentUser.mandateId, + "mandateId": self.mandateId, "sttLanguage": "de-DE", "ttsLanguage": "de-DE", "ttsVoice": "de-DE-Wavenet-A", @@ -524,21 +531,22 @@ class VoiceObjects: } -def getVoiceInterface(currentUser: User = None) -> VoiceObjects: +def getVoiceInterface(currentUser: User = None, mandateId: Optional[str] = None) -> VoiceObjects: """ Factory function to get or create Voice interface instance. Args: currentUser: User object for context (optional) + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. Returns: VoiceObjects instance """ - # For now, create a new instance each time - # In the future, this could be enhanced with singleton pattern per user + effectiveMandateId = str(mandateId) if mandateId else None + voiceInterface = VoiceObjects() if currentUser: - voiceInterface.setUserContext(currentUser) + voiceInterface.setUserContext(currentUser, mandateId=effectiveMandateId) return voiceInterface diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index e7902b0f..caa39859 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -3,20 +3,70 @@ """ Admin RBAC Roles Management routes. Provides endpoints for managing roles and role assignments to users. + +MULTI-TENANT: These are SYSTEM-LEVEL operations requiring isSysAdmin=true. +Roles are global system resources, not mandate-specific. +Role assignments are managed via UserMandateRole (not User.roleLabels). """ from fastapi import APIRouter, HTTPException, Depends, Query, Body, Path, Request -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Set import logging -from modules.auth import getCurrentUser, limiter +from modules.auth import limiter, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelRbac import Role -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole +from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) + +def _getUserRoleLabels(interface, userId: str) -> List[str]: + """ + Get role labels for a user from UserMandateRole (across all mandates). + + Args: + interface: Database interface + userId: User ID + + Returns: + List of role labels + """ + roleLabels: Set[str] = set() + + # Get all UserMandate records for this user + userMandates = interface.db.getRecordset(UserMandate, recordFilter={"userId": userId}) + + for um in userMandates: + userMandateId = um.get("id") + if not userMandateId: + continue + + # Get all UserMandateRole records for this membership + userMandateRoles = interface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": str(userMandateId)} + ) + + for umr in userMandateRoles: + roleId = umr.get("roleId") + if roleId: + # Get role by ID to get roleLabel + role = interface.getRole(str(roleId)) + if role: + roleLabels.add(role.roleLabel) + + return list(roleLabels) + + +def _hasRoleLabel(interface, userId: str, roleLabel: str) -> bool: + """ + Check if user has a specific role label (across all mandates). + """ + return roleLabel in _getUserRoleLabels(interface, userId) + router = APIRouter( prefix="/api/admin/rbac/roles", tags=["Admin RBAC Roles"], @@ -24,51 +74,27 @@ router = APIRouter( ) -def _ensureAdminAccess(currentUser: User) -> None: - """Ensure current user has admin access to RBAC roles management.""" - interface = getInterface(currentUser) - - # Check if user has admin or sysadmin role - roleLabels = currentUser.roleLabels or [] - if "sysadmin" not in roleLabels and "admin" not in roleLabels: - raise HTTPException( - status_code=403, - detail="Admin or sysadmin role required to manage RBAC roles" - ) - - # Additional RBAC check: verify user has permission to update UserInDB - # This is already covered by admin/sysadmin role check above, but we can add explicit RBAC check if needed - # For now, admin/sysadmin role check is sufficient - - @router.get("/", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") async def listRoles( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get list of all available roles with metadata. + MULTI-TENANT: SysAdmin-only (roles are system resources). Returns: - List of role dictionaries with role label, description, and user count """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get all roles from database dbRoles = interface.getAllRoles() - # Get all users to count role assignments - allUsers = interface.getUsers() - - # Count users per role - roleCounts: Dict[str, int] = {} - for user in allUsers: - for roleLabel in (user.roleLabels or []): - roleCounts[roleLabel] = roleCounts.get(roleLabel, 0) + 1 + # Count role assignments from UserMandateRole table + roleCounts = interface.countRoleAssignments() # Convert Role objects to dictionaries and add user counts result = [] @@ -77,22 +103,10 @@ async def listRoles( "id": role.id, "roleLabel": role.roleLabel, "description": role.description, - "userCount": roleCounts.get(role.roleLabel, 0), + "userCount": roleCounts.get(str(role.id), 0), "isSystemRole": role.isSystemRole }) - # Add any roles found in user assignments that don't exist in database - dbRoleLabels = {role.roleLabel for role in dbRoles} - for roleLabel, count in roleCounts.items(): - if roleLabel not in dbRoleLabels: - result.append({ - "id": None, - "roleLabel": roleLabel, - "description": {"en": f"Custom role: {roleLabel}", "fr": f"Rôle personnalisé : {roleLabel}"}, - "userCount": count, - "isSystemRole": False - }) - return result except HTTPException: @@ -109,19 +123,17 @@ async def listRoles( @limiter.limit("60/minute") async def getRoleOptions( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get role options for select dropdowns. - Returns roles in format suitable for frontend select components. + MULTI-TENANT: SysAdmin-only. Returns: - List of role option dictionaries with value and label """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get all roles from database dbRoles = interface.getAllRoles() @@ -153,10 +165,11 @@ async def getRoleOptions( async def createRole( request: Request, role: Role = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Create a new role. + MULTI-TENANT: SysAdmin-only (roles are system resources). Request Body: - role: Role object to create @@ -165,9 +178,7 @@ async def createRole( - Created role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() createdRole = interface.createRole(role) @@ -198,10 +209,11 @@ async def createRole( async def getRole( request: Request, roleId: str = Path(..., description="Role ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get a role by ID. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -210,9 +222,7 @@ async def getRole( - Role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() role = interface.getRole(roleId) if not role: @@ -244,10 +254,11 @@ async def updateRole( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update an existing role. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -259,9 +270,7 @@ async def updateRole( - Updated role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() updatedRole = interface.updateRole(roleId, role) @@ -292,10 +301,11 @@ async def updateRole( async def deleteRole( request: Request, roleId: str = Path(..., description="Role ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, str]: """ Delete a role. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -304,9 +314,7 @@ async def deleteRole( - Success message """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() success = interface.deleteRole(roleId) if not success: @@ -337,48 +345,50 @@ async def deleteRole( async def listUsersWithRoles( request: Request, roleLabel: Optional[str] = Query(None, description="Filter by role label"), - mandateId: Optional[str] = Query(None, description="Filter by mandate ID"), - currentUser: User = Depends(getCurrentUser) + mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), + context: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get list of users with their role assignments. + MULTI-TENANT: SysAdmin-only, can see all users across mandates. Query Parameters: - roleLabel: Optional filter by role label - - mandateId: Optional filter by mandate ID + - mandateId: Optional filter by mandate ID (via UserMandate table) Returns: - List of user dictionaries with role assignments """ try: - _ensureAdminAccess(currentUser) + interface = getRootInterface() - interface = getInterface(currentUser) + # Get all users (SysAdmin sees all) + users = interface.getUsers() - # Get users based on filters + # Filter by mandate if specified (via UserMandate table) if mandateId: - # Filter by mandate (if user has permission) - users = interface.getUsers() - users = [u for u in users if u.mandateId == mandateId] - else: - users = interface.getUsers() + from modules.datamodels.datamodelMembership import UserMandate + userMandates = interface.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) + mandateUserIds = {str(um["userId"]) for um in userMandates} + users = [u for u in users if str(u.id) in mandateUserIds] - # Filter by role if specified + # Filter by role if specified (via UserMandateRole) if roleLabel: - users = [u for u in users if roleLabel in (u.roleLabels or [])] + users = [u for u in users if _hasRoleLabel(interface, str(u.id), roleLabel)] # Format response result = [] for user in users: + userRoleLabels = _getUserRoleLabels(interface, str(user.id)) result.append({ "id": user.id, "username": user.username, "email": user.email, "fullName": user.fullName, - "mandateId": user.mandateId, + "isSysAdmin": user.isSysAdmin, "enabled": user.enabled, - "roleLabels": user.roleLabels or [], - "roleCount": len(user.roleLabels or []) + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) }) return result @@ -398,10 +408,11 @@ async def listUsersWithRoles( async def getUserRoles( request: Request, userId: str = Path(..., description="User ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get role assignments for a specific user. + MULTI-TENANT: SysAdmin-only. Path Parameters: - userId: User ID @@ -410,9 +421,7 @@ async def getUserRoles( - User dictionary with role assignments """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get user user = interface.getUser(userId) @@ -422,15 +431,16 @@ async def getUserRoles( detail=f"User {userId} not found" ) + userRoleLabels = _getUserRoleLabels(interface, str(user.id)) return { "id": user.id, "username": user.username, "email": user.email, "fullName": user.fullName, - "mandateId": user.mandateId, + "isSysAdmin": user.isSysAdmin, "enabled": user.enabled, - "roleLabels": user.roleLabels or [], - "roleCount": len(user.roleLabels or []) + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) } except HTTPException: @@ -448,25 +458,24 @@ async def getUserRoles( async def updateUserRoles( request: Request, userId: str = Path(..., description="User ID"), - roleLabels: List[str] = Body(..., description="List of role labels to assign"), - currentUser: User = Depends(getCurrentUser) + newRoleLabels: List[str] = Body(..., description="List of role labels to assign"), + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update role assignments for a specific user. + MULTI-TENANT: SysAdmin-only. Updates roles in user's first mandate. Path Parameters: - userId: User ID Request Body: - - roleLabels: List of role labels to assign (e.g., ["admin", "user"]) + - newRoleLabels: List of role labels to assign (e.g., ["admin", "user"]) Returns: - Updated user dictionary with role assignments """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get user user = interface.getUser(userId) @@ -478,28 +487,57 @@ async def updateUserRoles( # Validate role labels (basic validation - check against standard roles) standardRoles = ["sysadmin", "admin", "user", "viewer"] - for roleLabel in roleLabels: + for roleLabel in newRoleLabels: if roleLabel not in standardRoles: logger.warning(f"Non-standard role label assigned: {roleLabel}") - # Update user roles - userData = { - "roleLabels": roleLabels - } + # Get user's first mandate (for role assignment) + userMandates = interface.db.getRecordset(UserMandate, recordFilter={"userId": userId}) + if not userMandates: + raise HTTPException( + status_code=400, + detail=f"User {userId} has no mandate memberships. Add to mandate first." + ) - updatedUser = interface.updateUser(userId, userData) + userMandateId = str(userMandates[0].get("id")) - logger.info(f"Updated roles for user {userId}: {roleLabels}") + # Get current roles for this mandate + existingRoles = interface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId} + ) + existingRoleIds = {str(r.get("roleId")) for r in existingRoles} + # Convert roleLabels to roleIds + newRoleIds = set() + for roleLabel in newRoleLabels: + role = interface.getRoleByLabel(roleLabel) + if role: + newRoleIds.add(str(role.id)) + + # Remove roles that are no longer needed + for existingRole in existingRoles: + if str(existingRole.get("roleId")) not in newRoleIds: + interface.db.recordDelete(UserMandateRole, str(existingRole.get("id"))) + + # Add new roles + for roleId in newRoleIds: + if roleId not in existingRoleIds: + newRole = UserMandateRole(userMandateId=userMandateId, roleId=roleId) + interface.db.recordCreate(UserMandateRole, newRole.model_dump()) + + logger.info(f"Updated roles for user {userId}: {newRoleLabels} by SysAdmin {context.user.id}") + + userRoleLabels = _getUserRoleLabels(interface, userId) return { - "id": updatedUser.id, - "username": updatedUser.username, - "email": updatedUser.email, - "fullName": updatedUser.fullName, - "mandateId": updatedUser.mandateId, - "enabled": updatedUser.enabled, - "roleLabels": updatedUser.roleLabels or [], - "roleCount": len(updatedUser.roleLabels or []) + "id": user.id, + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "isSysAdmin": user.isSysAdmin, + "enabled": user.enabled, + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) } except HTTPException: @@ -518,10 +556,11 @@ async def addUserRole( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to add"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Add a role to a user (if not already assigned). + MULTI-TENANT: SysAdmin-only. Adds role to user's first mandate. Path Parameters: - userId: User ID @@ -531,9 +570,7 @@ async def addUserRole( - Updated user dictionary with role assignments """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get user user = interface.getUser(userId) @@ -543,33 +580,46 @@ async def addUserRole( detail=f"User {userId} not found" ) - # Get current roles - currentRoles = list(user.roleLabels or []) + # Get role by label + role = interface.getRoleByLabel(roleLabel) + if not role: + raise HTTPException( + status_code=404, + detail=f"Role '{roleLabel}' not found" + ) - # Add role if not already present - if roleLabel not in currentRoles: - currentRoles.append(roleLabel) - - # Update user roles - userData = { - "roleLabels": currentRoles - } - - updatedUser = interface.updateUser(userId, userData) - - logger.info(f"Added role {roleLabel} to user {userId}") - else: - updatedUser = user + # Get user's first mandate + userMandates = interface.db.getRecordset(UserMandate, recordFilter={"userId": userId}) + if not userMandates: + raise HTTPException( + status_code=400, + detail=f"User {userId} has no mandate memberships. Add to mandate first." + ) + userMandateId = str(userMandates[0].get("id")) + + # Check if role is already assigned + existingAssignment = interface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId, "roleId": str(role.id)} + ) + + if not existingAssignment: + # Add the role + newRole = UserMandateRole(userMandateId=userMandateId, roleId=str(role.id)) + interface.db.recordCreate(UserMandateRole, newRole.model_dump()) + logger.info(f"Added role {roleLabel} to user {userId} by SysAdmin {context.user.id}") + + userRoleLabels = _getUserRoleLabels(interface, userId) return { - "id": updatedUser.id, - "username": updatedUser.username, - "email": updatedUser.email, - "fullName": updatedUser.fullName, - "mandateId": updatedUser.mandateId, - "enabled": updatedUser.enabled, - "roleLabels": updatedUser.roleLabels or [], - "roleCount": len(updatedUser.roleLabels or []) + "id": user.id, + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "isSysAdmin": user.isSysAdmin, + "enabled": user.enabled, + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) } except HTTPException: @@ -588,10 +638,11 @@ async def removeUserRole( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to remove"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Remove a role from a user. + MULTI-TENANT: SysAdmin-only. Removes role from all user's mandates. Path Parameters: - userId: User ID @@ -601,9 +652,7 @@ async def removeUserRole( - Updated user dictionary with role assignments """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get user user = interface.getUser(userId) @@ -613,38 +662,44 @@ async def removeUserRole( detail=f"User {userId} not found" ) - # Get current roles - currentRoles = list(user.roleLabels or []) + # Get role by label + role = interface.getRoleByLabel(roleLabel) + if not role: + raise HTTPException( + status_code=404, + detail=f"Role '{roleLabel}' not found" + ) - # Remove role if present - if roleLabel in currentRoles: - currentRoles.remove(roleLabel) - - # Ensure user has at least one role (default to "user") - if not currentRoles: - currentRoles = ["user"] - logger.warning(f"User {userId} had all roles removed, defaulting to 'user' role") - - # Update user roles - userData = { - "roleLabels": currentRoles - } - - updatedUser = interface.updateUser(userId, userData) - - logger.info(f"Removed role {roleLabel} from user {userId}") - else: - updatedUser = user + # Remove role from all user's mandates + userMandates = interface.db.getRecordset(UserMandate, recordFilter={"userId": userId}) + roleRemoved = False + for um in userMandates: + userMandateId = str(um.get("id")) + + # Find and delete the role assignment + assignments = interface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId, "roleId": str(role.id)} + ) + + for assignment in assignments: + interface.db.recordDelete(UserMandateRole, str(assignment.get("id"))) + roleRemoved = True + + if roleRemoved: + logger.info(f"Removed role {roleLabel} from user {userId} by SysAdmin {context.user.id}") + + userRoleLabels = _getUserRoleLabels(interface, userId) return { - "id": updatedUser.id, - "username": updatedUser.username, - "email": updatedUser.email, - "fullName": updatedUser.fullName, - "mandateId": updatedUser.mandateId, - "enabled": updatedUser.enabled, - "roleLabels": updatedUser.roleLabels or [], - "roleCount": len(updatedUser.roleLabels or []) + "id": user.id, + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "isSysAdmin": user.isSysAdmin, + "enabled": user.enabled, + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) } except HTTPException: @@ -662,49 +717,69 @@ async def removeUserRole( async def getUsersWithRole( request: Request, roleLabel: str = Path(..., description="Role label"), - mandateId: Optional[str] = Query(None, description="Filter by mandate ID"), - currentUser: User = Depends(getCurrentUser) + mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), + context: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get all users with a specific role. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleLabel: Role label Query Parameters: - - mandateId: Optional filter by mandate ID + - mandateId: Optional filter by mandate ID (via UserMandate table) Returns: - List of users with the specified role """ try: - _ensureAdminAccess(currentUser) + interface = getRootInterface() - interface = getInterface(currentUser) + # Get role by label + role = interface.getRoleByLabel(roleLabel) + if not role: + raise HTTPException( + status_code=404, + detail=f"Role '{roleLabel}' not found" + ) - # Get all users - users = interface.getUsers() + # Get all UserMandateRole assignments for this role + roleAssignments = interface.db.getRecordset( + UserMandateRole, + recordFilter={"roleId": str(role.id)} + ) - # Filter by role - users = [u for u in users if roleLabel in (u.roleLabels or [])] + # Get unique userMandateIds + userMandateIds = {str(ra.get("userMandateId")) for ra in roleAssignments} - # Filter by mandate if specified - if mandateId: - users = [u for u in users if u.mandateId == mandateId] + # Get userIds from UserMandate records + userIds: Set[str] = set() + for userMandateId in userMandateIds: + umRecords = interface.db.getRecordset(UserMandate, recordFilter={"id": userMandateId}) + if umRecords: + um = umRecords[0] + # Filter by mandate if specified + if mandateId and str(um.get("mandateId")) != mandateId: + continue + userIds.add(str(um.get("userId"))) - # Format response + # Get users and format response result = [] - for user in users: - result.append({ - "id": user.id, - "username": user.username, - "email": user.email, - "fullName": user.fullName, - "mandateId": user.mandateId, - "enabled": user.enabled, - "roleLabels": user.roleLabels or [], - "roleCount": len(user.roleLabels or []) - }) + for userId in userIds: + user = interface.getUser(userId) + if user: + userRoleLabels = _getUserRoleLabels(interface, userId) + result.append({ + "id": user.id, + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "isSysAdmin": user.isSysAdmin, + "enabled": user.enabled, + "roleLabels": userRoleLabels, + "roleCount": len(userRoleLabels) + }) return result diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 12ed265c..3b5515df 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -117,7 +117,7 @@ async def create_automation( chatInterface = getChatInterface(currentUser) automationData = automation.model_dump() created = chatInterface.createAutomationDefinition(automationData) - return AutomationDefinition(**created) + return created except HTTPException: raise except Exception as e: @@ -171,7 +171,7 @@ async def get_automation( detail=f"Automation {automationId} not found" ) - return AutomationDefinition(**automation) + return automation except HTTPException: raise except Exception as e: @@ -194,7 +194,7 @@ async def update_automation( chatInterface = getChatInterface(currentUser) automationData = automation.model_dump() updated = chatInterface.updateAutomationDefinition(automationId, automationData) - return AutomationDefinition(**updated) + return updated except HTTPException: raise except PermissionError as e: diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 8a0c310a..a118869a 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -3,6 +3,10 @@ """ Mandate routes for the backend API. Implements the endpoints for mandate management. + +MULTI-TENANT: +- Mandate CRUD is SysAdmin-only (mandates are system resources) +- User management within mandates is Mandate-Admin (add/remove users) """ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response, Query @@ -10,18 +14,53 @@ from typing import List, Dict, Any, Optional from fastapi import status import logging import json +from pydantic import BaseModel, Field # Import auth module -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, requireSysAdmin, getRequestContext, RequestContext # Import interfaces import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects from modules.shared.attributeUtils import getModelAttributeDefinitions +from modules.shared.auditLogger import audit_logger # Import the model classes from modules.datamodels.datamodelUam import Mandate, User +from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole +from modules.datamodels.datamodelRbac import Role from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict + +# ============================================================================= +# Request/Response Models for User Management +# ============================================================================= + +class UserMandateCreate(BaseModel): + """Request model for adding a user to a mandate""" + targetUserId: str = Field(..., description="User ID to add to the mandate") + roleIds: List[str] = Field(..., description="Role IDs to assign to the user") + + +class UserMandateResponse(BaseModel): + """Response model for user mandate membership""" + userMandateId: str + userId: str + mandateId: str + roleIds: List[str] + enabled: bool + + +class MandateUserInfo(BaseModel): + """User info within a mandate context""" + userId: str + username: str + email: Optional[str] + firstname: Optional[str] + lastname: Optional[str] + userMandateId: str + roleIds: List[str] + enabled: bool + # Configure logger logger = logging.getLogger(__name__) @@ -40,10 +79,11 @@ router = APIRouter( async def get_mandates( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> PaginatedResponse[Mandate]: """ Get mandates with optional pagination, sorting, and filtering. + MULTI-TENANT: SysAdmin-only (mandates are system resources). Query Parameters: - pagination: JSON-encoded PaginationParams object, or None for no pagination @@ -67,7 +107,7 @@ async def get_mandates( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getRootInterface() result = appInterface.getAllMandates(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -103,11 +143,14 @@ async def get_mandates( async def get_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Mandate: - """Get a specific mandate by ID""" + """ + Get a specific mandate by ID. + MULTI-TENANT: SysAdmin-only. + """ try: - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getRootInterface() mandate = appInterface.getMandate(mandateId) if not mandate: @@ -131,9 +174,12 @@ async def get_mandate( async def create_mandate( request: Request, mandateData: dict = Body(..., description="Mandate data with at least 'name' field"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Mandate: - """Create a new mandate""" + """ + Create a new mandate. + MULTI-TENANT: SysAdmin-only. + """ try: logger.debug(f"Creating mandate with data: {mandateData}") @@ -148,7 +194,7 @@ async def create_mandate( # Get optional fields with defaults language = mandateData.get('language', 'en') - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getRootInterface() # Create mandate newMandate = appInterface.createMandate( @@ -161,6 +207,8 @@ async def create_mandate( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create mandate" ) + + logger.info(f"Mandate {newMandate.id} created by SysAdmin {context.user.id}") return newMandate except HTTPException: @@ -178,13 +226,16 @@ async def update_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate to update"), mandateData: dict = Body(..., description="Mandate update data"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Mandate: - """Update an existing mandate""" + """ + Update an existing mandate. + MULTI-TENANT: SysAdmin-only. + """ try: logger.debug(f"Updating mandate {mandateId} with data: {mandateData}") - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -202,6 +253,8 @@ async def update_mandate( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update mandate" ) + + logger.info(f"Mandate {mandateId} updated by SysAdmin {context.user.id}") return updatedMandate except HTTPException: @@ -218,11 +271,14 @@ async def update_mandate( async def delete_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate to delete"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: - """Delete a mandate""" + """ + Delete a mandate. + MULTI-TENANT: SysAdmin-only. + """ try: - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -231,6 +287,13 @@ async def delete_mandate( status_code=status.HTTP_404_NOT_FOUND, detail=f"Mandate {mandateId} not found" ) + + # MULTI-TENANT: Delete all UserMandate entries for this mandate first + from modules.datamodels.datamodelMembership import UserMandate + userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) + for um in userMandates: + appInterface.db.deleteRecord(UserMandate, um["id"]) + logger.info(f"Deleted {len(userMandates)} UserMandate entries for mandate {mandateId}") # Delete mandate try: @@ -240,6 +303,8 @@ async def delete_mandate( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) ) + + logger.info(f"Mandate {mandateId} deleted by SysAdmin {context.user.id}") return {"message": f"Mandate {mandateId} deleted successfully"} except HTTPException: @@ -250,3 +315,456 @@ async def delete_mandate( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to delete mandate: {str(e)}" ) + + +# ============================================================================= +# User Management within Mandates (Mandate-Admin) +# ============================================================================= + +@router.get("/{mandateId}/users", response_model=List[MandateUserInfo]) +@limiter.limit("60/minute") +async def listMandateUsers( + request: Request, + mandateId: str = Path(..., description="ID of the mandate"), + context: RequestContext = Depends(getRequestContext) +) -> List[MandateUserInfo]: + """ + List all users in a mandate. + + Requires Mandate-Admin role or SysAdmin. + """ + # Check permission + if not _hasMandateAdminRole(context, mandateId) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required" + ) + + try: + rootInterface = interfaceDbAppObjects.getRootInterface() + + # Verify mandate exists + mandate = rootInterface.getMandate(mandateId) + if not mandate: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Mandate {mandateId} not found" + ) + + # Get all UserMandate entries for this mandate + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"mandateId": mandateId} + ) + + result = [] + for um in userMandates: + # Get user info + user = rootInterface.getUserById(um.get("userId")) + if not user: + continue + + # Get roles for this membership + roleIds = rootInterface.getRoleIdsForUserMandate(um.get("id")) + + result.append(MandateUserInfo( + userId=str(user.id), + username=user.username, + email=user.email, + firstname=user.firstname, + lastname=user.lastname, + userMandateId=um.get("id"), + roleIds=roleIds, + enabled=um.get("enabled", True) + )) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing users for mandate {mandateId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list users: {str(e)}" + ) + + +@router.post("/{mandateId}/users", response_model=UserMandateResponse) +@limiter.limit("30/minute") +async def addUserToMandate( + request: Request, + mandateId: str = Path(..., description="ID of the mandate"), + data: UserMandateCreate = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> UserMandateResponse: + """ + Add a user to a mandate with specified roles. + + Requires Mandate-Admin role. + SysAdmin cannot add themselves (Self-Eskalation Prevention). + + Args: + mandateId: Target mandate ID + data: User ID and role IDs to assign + """ + # 1. SysAdmin Self-Eskalation Prevention + if context.isSysAdmin and data.targetUserId == str(context.user.id): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="SysAdmin cannot add themselves to a mandate. A Mandate-Admin must grant access." + ) + + # 2. Check Mandate-Admin permission + if not _hasMandateAdminRole(context, mandateId): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to add users" + ) + + try: + rootInterface = interfaceDbAppObjects.getRootInterface() + + # 3. Verify mandate exists + mandate = rootInterface.getMandate(mandateId) + if not mandate: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Mandate {mandateId} not found" + ) + + # 4. Verify target user exists + targetUser = rootInterface.getUserById(data.targetUserId) + if not targetUser: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User {data.targetUserId} not found" + ) + + # 5. Check if user is already a member + existingMembership = rootInterface.getUserMandate(data.targetUserId, mandateId) + if existingMembership: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"User {data.targetUserId} is already a member of this mandate" + ) + + # 6. Validate roles (must exist and belong to this mandate or be global) + for roleId in data.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roleRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Role {roleId} not found" + ) + role = roleRecords[0] + roleMandateId = role.get("mandateId") + if roleMandateId and str(roleMandateId) != str(mandateId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role {roleId} belongs to a different mandate" + ) + + # 7. Create UserMandate + userMandate = rootInterface.createUserMandate( + userId=data.targetUserId, + mandateId=mandateId, + roleIds=data.roleIds + ) + + # 8. Audit + audit_logger.logSecurityEvent( + userId=str(context.user.id), + mandateId=mandateId, + action="user_added_to_mandate", + details=f"targetUser={data.targetUserId}, roles={data.roleIds}" + ) + + logger.info( + f"User {context.user.id} added user {data.targetUserId} to mandate {mandateId} " + f"with roles {data.roleIds}" + ) + + return UserMandateResponse( + userMandateId=str(userMandate.id), + userId=data.targetUserId, + mandateId=mandateId, + roleIds=data.roleIds, + enabled=True + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding user to mandate: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to add user to mandate: {str(e)}" + ) + + +@router.delete("/{mandateId}/users/{targetUserId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +async def removeUserFromMandate( + request: Request, + mandateId: str = Path(..., description="ID of the mandate"), + targetUserId: str = Path(..., description="ID of the user to remove"), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, str]: + """ + Remove a user from a mandate. + + Requires Mandate-Admin role. + Cannot remove the last admin from a mandate (orphan prevention). + + Args: + mandateId: Target mandate ID + targetUserId: User ID to remove + """ + # Check Mandate-Admin permission + if not _hasMandateAdminRole(context, mandateId): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required" + ) + + try: + rootInterface = interfaceDbAppObjects.getRootInterface() + + # Verify mandate exists + mandate = rootInterface.getMandate(mandateId) + if not mandate: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Mandate {mandateId} not found" + ) + + # Get user's membership + membership = rootInterface.getUserMandate(targetUserId, mandateId) + if not membership: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User {targetUserId} is not a member of this mandate" + ) + + # Check if this is the last admin (orphan prevention) + if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot remove the last admin from a mandate. Assign another admin first." + ) + + # Delete UserMandate (CASCADE will delete UserMandateRole entries) + rootInterface.deleteUserMandate(targetUserId, mandateId) + + # Audit + audit_logger.logSecurityEvent( + userId=str(context.user.id), + mandateId=mandateId, + action="user_removed_from_mandate", + details=f"targetUser={targetUserId}" + ) + + logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {mandateId}") + + return {"message": "User removed from mandate", "userId": targetUserId} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error removing user from mandate: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to remove user from mandate: {str(e)}" + ) + + +@router.put("/{mandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) +@limiter.limit("30/minute") +async def updateUserRolesInMandate( + request: Request, + mandateId: str = Path(..., description="ID of the mandate"), + targetUserId: str = Path(..., description="ID of the user"), + roleIds: List[str] = Body(..., description="New role IDs to assign"), + context: RequestContext = Depends(getRequestContext) +) -> UserMandateResponse: + """ + Update a user's roles within a mandate. + + Replaces all existing roles with the new set. + Requires Mandate-Admin role. + + Args: + mandateId: Target mandate ID + targetUserId: User ID to update + roleIds: New set of role IDs + """ + # Check Mandate-Admin permission + if not _hasMandateAdminRole(context, mandateId): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required" + ) + + try: + rootInterface = interfaceDbAppObjects.getRootInterface() + + # Get user's membership + membership = rootInterface.getUserMandate(targetUserId, mandateId) + if not membership: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User {targetUserId} is not a member of this mandate" + ) + + # Validate new roles + for roleId in roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roleRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Role {roleId} not found" + ) + role = roleRecords[0] + roleMandateId = role.get("mandateId") + if roleMandateId and str(roleMandateId) != str(mandateId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role {roleId} belongs to a different mandate" + ) + + # Check if removing admin role would leave mandate without admins + currentRoleIds = rootInterface.getRoleIdsForUserMandate(str(membership.id)) + isCurrentlyAdmin = _hasAdminRoleInList(rootInterface, currentRoleIds, mandateId) + willBeAdmin = _hasAdminRoleInList(rootInterface, roleIds, mandateId) + + if isCurrentlyAdmin and not willBeAdmin: + if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot remove admin role from the last admin. Assign another admin first." + ) + + # Remove existing role assignments + existingRoles = rootInterface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": str(membership.id)} + ) + for er in existingRoles: + rootInterface.db.recordDelete(UserMandateRole, er.get("id")) + + # Add new role assignments + for roleId in roleIds: + rootInterface.addRoleToUserMandate(str(membership.id), roleId) + + # Audit + audit_logger.logSecurityEvent( + userId=str(context.user.id), + mandateId=mandateId, + action="user_roles_updated_in_mandate", + details=f"targetUser={targetUserId}, newRoles={roleIds}" + ) + + logger.info( + f"User {context.user.id} updated roles for user {targetUserId} " + f"in mandate {mandateId} to {roleIds}" + ) + + return UserMandateResponse( + userMandateId=str(membership.id), + userId=targetUserId, + mandateId=mandateId, + roleIds=roleIds, + enabled=membership.enabled + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating user roles in mandate: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update user roles: {str(e)}" + ) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _hasMandateAdminRole(context: RequestContext, mandateId: str) -> bool: + """ + Check if the user has mandate admin role for the specified mandate. + """ + if context.isSysAdmin: + return True + + # Must be in the same mandate context + if str(context.mandateId) != str(mandateId): + return False + + if not context.roleIds: + return False + + try: + rootInterface = interfaceDbAppObjects.getRootInterface() + + for roleId in context.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + # Admin role at mandate level (not feature-instance level) + if roleLabel == "admin" and role.get("mandateId") and not role.get("featureInstanceId"): + return True + + return False + + except Exception as e: + logger.error(f"Error checking mandate admin role: {e}") + return False + + +def _isLastMandateAdmin(interface, mandateId: str, excludeUserId: str) -> bool: + """ + Check if excluding this user would leave the mandate without any admins. + """ + try: + # Get all UserMandates for this mandate + userMandates = interface.db.getRecordset( + UserMandate, + recordFilter={"mandateId": mandateId, "enabled": True} + ) + + adminCount = 0 + for um in userMandates: + if str(um.get("userId")) == str(excludeUserId): + continue + + # Check if this user has admin role + roleIds = interface.getRoleIdsForUserMandate(um.get("id")) + if _hasAdminRoleInList(interface, roleIds, mandateId): + adminCount += 1 + + return adminCount == 0 + + except Exception as e: + logger.error(f"Error checking last admin: {e}") + return True # Fail-safe: assume they're the last admin + + +def _hasAdminRoleInList(interface, roleIds: List[str], mandateId: str) -> bool: + """ + Check if any of the role IDs is an admin role for the mandate. + """ + for roleId in roleIds: + roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + roleMandateId = role.get("mandateId") + # Admin role at mandate level + if roleLabel == "admin" and (not roleMandateId or str(roleMandateId) == str(mandateId)): + if not role.get("featureInstanceId"): + return True + return False diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 525651c7..ae6ab70a 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -3,6 +3,10 @@ """ User routes for the backend API. Implements the endpoints for user management. + +MULTI-TENANT: User management requires RequestContext. +- mandateId from X-Mandate-Id header determines which users are visible +- SysAdmin can see all users across mandates """ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response, Query @@ -13,7 +17,7 @@ import json # Import interfaces and models import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects -from modules.auth import getCurrentUser, limiter +from modules.auth import limiter, getRequestContext, RequestContext # Import the attribute definition and helper functions from modules.datamodels.datamodelUam import User @@ -32,19 +36,19 @@ router = APIRouter( @limiter.limit("30/minute") async def get_users( request: Request, - mandateId: Optional[str] = Query(None, description="Mandate ID to filter users"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[User]: """ Get users with optional pagination, sorting, and filtering. + MULTI-TENANT: mandateId from X-Mandate-Id header determines scope. + SysAdmin without mandateId sees all users. Query Parameters: - - mandateId: Optional mandate ID to filter users - pagination: JSON-encoded PaginationParams object, or None for no pagination Examples: - - GET /api/users/ (no pagination - returns all users) + - GET /api/users/ (no pagination - returns all users in mandate) - GET /api/users/?pagination={"page":1,"pageSize":10,"sort":[]} """ try: @@ -62,30 +66,77 @@ async def get_users( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getInterface(currentUser) - # If mandateId is provided, use it, otherwise use the current user's mandate - targetMandateId = mandateId or currentUser.mandateId - # Get users with optional pagination - result = appInterface.getUsersByMandate(targetMandateId, pagination=paginationParams) + appInterface = interfaceDbAppObjects.getInterface(context.user) - # If pagination was requested, result is PaginatedResult - # If no pagination, result is List[User] - if paginationParams: - return PaginatedResponse( - items=result.items, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=result.totalItems, - totalPages=result.totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters + # MULTI-TENANT: Use mandateId from context (header) + # SysAdmin without mandateId can see all users + if context.mandateId: + # Get users for specific mandate via UserMandate table + from modules.datamodels.datamodelMembership import UserMandate + userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": str(context.mandateId)}) + userIds = [str(um["userId"]) for um in userMandates] + + # Get all users and filter by mandate membership + allUsers = appInterface.getUsers() + users = [u for u in allUsers if str(u.id) in userIds] + + # Apply pagination manually if needed + if paginationParams: + totalItems = len(users) + import math + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + paginatedUsers = users[startIdx:endIdx] + + return PaginatedResponse( + items=paginatedUsers, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + else: + return PaginatedResponse( + items=users, + pagination=None + ) + elif context.isSysAdmin: + # SysAdmin without mandateId sees all users + result = appInterface.getUsers() + if paginationParams: + totalItems = len(result) + import math + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + paginatedUsers = result[startIdx:endIdx] + + return PaginatedResponse( + items=paginatedUsers, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + else: + return PaginatedResponse( + items=result, + pagination=None ) - ) else: - return PaginatedResponse( - items=result, - pagination=None + # Non-SysAdmin without mandateId - should not happen (getRequestContext enforces) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" ) except HTTPException: raise @@ -101,11 +152,14 @@ async def get_users( async def get_user( request: Request, userId: str = Path(..., description="ID of the user"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> User: - """Get a specific user by ID""" + """ + Get a specific user by ID. + MULTI-TENANT: User must be in the same mandate (via UserMandate) or caller is SysAdmin. + """ try: - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getInterface(context.user) # Get user without filtering by enabled status user = appInterface.getUser(userId) @@ -114,6 +168,19 @@ async def get_user( status_code=status.HTTP_404_NOT_FOUND, detail=f"User with ID {userId} not found" ) + + # MULTI-TENANT: Verify user is in the same mandate (unless SysAdmin) + if context.mandateId and not context.isSysAdmin: + from modules.datamodels.datamodelMembership import UserMandate + userMandate = appInterface.db.getRecordset(UserMandate, recordFilter={ + "userId": userId, + "mandateId": str(context.mandateId) + }) + if not userMandate: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="User not in your mandate" + ) return user except HTTPException: @@ -131,10 +198,13 @@ async def create_user( request: Request, user_data: User = Body(...), password: Optional[str] = Body(None, embed=True), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> User: - """Create a new user""" - appInterface = interfaceDbAppObjects.getInterface(currentUser) + """ + Create a new user. + MULTI-TENANT: User is created and automatically added to the current mandate. + """ + appInterface = interfaceDbAppObjects.getInterface(context.user) # Extract fields from User model and call createUser with individual parameters from modules.datamodels.datamodelUam import AuthAuthority @@ -145,10 +215,22 @@ async def create_user( fullName=user_data.fullName, language=user_data.language, enabled=user_data.enabled, - roleLabels=user_data.roleLabels if user_data.roleLabels else ["user"], authenticationAuthority=user_data.authenticationAuthority ) + # MULTI-TENANT: Add user to current mandate via UserMandate with default "user" role + if context.mandateId: + # Get "user" role ID + userRole = appInterface.getRoleByLabel("user") + roleIds = [str(userRole.id)] if userRole else [] + + appInterface.createUserMandate( + userId=str(newUser.id), + mandateId=str(context.mandateId), + roleIds=roleIds + ) + logger.info(f"Created UserMandate for user {newUser.id} in mandate {context.mandateId}") + return newUser @router.put("/{userId}", response_model=User) @@ -157,10 +239,13 @@ async def update_user( request: Request, userId: str = Path(..., description="ID of the user to update"), userData: User = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> User: - """Update an existing user""" - appInterface = interfaceDbAppObjects.getInterface(currentUser) + """ + Update an existing user. + MULTI-TENANT: Can only update users in the same mandate (unless SysAdmin). + """ + appInterface = interfaceDbAppObjects.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) @@ -170,6 +255,19 @@ async def update_user( detail=f"User with ID {userId} not found" ) + # MULTI-TENANT: Verify user is in the same mandate (unless SysAdmin) + if context.mandateId and not context.isSysAdmin: + from modules.datamodels.datamodelMembership import UserMandate + userMandate = appInterface.db.getRecordset(UserMandate, recordFilter={ + "userId": userId, + "mandateId": str(context.mandateId) + }) + if not userMandate: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Cannot update user outside your mandate" + ) + # Update user updatedUser = appInterface.updateUser(userId, userData) @@ -187,19 +285,22 @@ async def reset_user_password( request: Request, userId: str = Path(..., description="ID of the user to reset password for"), newPassword: str = Body(..., embed=True), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Reset user password (Admin only)""" + """ + Reset user password (Admin only). + MULTI-TENANT: Can only reset passwords for users in the same mandate (unless SysAdmin). + """ try: # Check if current user is admin - if "admin" not in (currentUser.roleLabels or []) and "sysadmin" not in (currentUser.roleLabels or []): + if not context.isSysAdmin: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Only administrators can reset passwords" ) # Get user interface - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getInterface(context.user) # Get target user target_user = appInterface.getUserById(userId) @@ -209,6 +310,19 @@ async def reset_user_password( detail="User not found" ) + # MULTI-TENANT: Verify user is in the same mandate (unless SysAdmin) + if context.mandateId and not context.isSysAdmin: + from modules.datamodels.datamodelMembership import UserMandate + userMandate = appInterface.db.getRecordset(UserMandate, recordFilter={ + "userId": userId, + "mandateId": str(context.mandateId) + }) + if not userMandate: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Cannot reset password for user outside your mandate" + ) + # Validate password strength if len(newPassword) < 8: raise HTTPException( @@ -231,7 +345,7 @@ async def reset_user_password( userId=userId, authority=None, # Revoke all authorities mandateId=None, # Revoke across all mandates - revokedBy=currentUser.id, + revokedBy=context.user.id, reason="password_reset" ) logger.info(f"Revoked {revoked_count} tokens for user {userId} after password reset") @@ -243,8 +357,8 @@ async def reset_user_password( try: from modules.shared.auditLogger import audit_logger audit_logger.logSecurityEvent( - userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + userId=str(context.user.id), + mandateId=str(context.mandateId) if context.mandateId else "system", action="password_reset", details=f"Reset password for user {userId}" ) @@ -271,15 +385,18 @@ async def change_password( request: Request, currentPassword: str = Body(..., embed=True), newPassword: str = Body(..., embed=True), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Change current user's password""" + """ + Change current user's password. + MULTI-TENANT: User changes their own password (no mandate restriction). + """ try: # Get user interface - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getInterface(context.user) # Verify current password - if not appInterface.verifyPassword(currentPassword, currentUser.passwordHash): + if not appInterface.verifyPassword(currentPassword, context.user.passwordHash): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Current password is incorrect" @@ -293,7 +410,7 @@ async def change_password( ) # Change password - success = appInterface.resetUserPassword(str(currentUser.id), newPassword) + success = appInterface.resetUserPassword(str(context.user.id), newPassword) if not success: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -304,23 +421,23 @@ async def change_password( try: from modules.datamodels.datamodelUam import AuthAuthority revoked_count = appInterface.revokeTokensByUser( - userId=str(currentUser.id), + userId=str(context.user.id), authority=None, # Revoke all authorities mandateId=None, # Revoke across all mandates - revokedBy=currentUser.id, + revokedBy=context.user.id, reason="password_change" ) - logger.info(f"Revoked {revoked_count} tokens for user {currentUser.id} after password change") + logger.info(f"Revoked {revoked_count} tokens for user {context.user.id} after password change") except Exception as e: - logger.error(f"Failed to revoke tokens after password change for user {currentUser.id}: {str(e)}") + logger.error(f"Failed to revoke tokens after password change for user {context.user.id}: {str(e)}") # Don't fail the password change if token revocation fails # Log password change try: from modules.shared.auditLogger import audit_logger audit_logger.logSecurityEvent( - userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + userId=str(context.user.id), + mandateId=str(context.mandateId) if context.mandateId else "system", action="password_change", details="User changed their own password" ) @@ -346,9 +463,11 @@ async def sendPasswordLink( request: Request, userId: str = Path(..., description="ID of the user to send password setup link"), frontendUrl: str = Body(..., embed=True), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Send password setup/reset link to a user (admin function). + """ + Send password setup/reset link to a user (admin function). + MULTI-TENANT: Can only send to users in the same mandate (unless SysAdmin). This allows admins to send a magic link to users to set or reset their password. Used when creating users without password or to help users who forgot their password. @@ -362,7 +481,7 @@ async def sendPasswordLink( from modules.interfaces.interfaceDbAppObjects import getRootInterface # Get user interface - appInterface = interfaceDbAppObjects.getInterface(currentUser) + appInterface = interfaceDbAppObjects.getInterface(context.user) # Get target user targetUser = appInterface.getUser(userId) @@ -372,6 +491,19 @@ async def sendPasswordLink( detail="User not found" ) + # MULTI-TENANT: Verify user is in the same mandate (unless SysAdmin) + if context.mandateId and not context.isSysAdmin: + from modules.datamodels.datamodelMembership import UserMandate + userMandate = appInterface.db.getRecordset(UserMandate, recordFilter={ + "userId": userId, + "mandateId": str(context.mandateId) + }) + if not userMandate: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Cannot send password link to user outside your mandate" + ) + # Check if user has an email if not targetUser.email: raise HTTPException( @@ -440,15 +572,15 @@ Falls Sie diese Anforderung nicht erwartet haben, kontaktieren Sie bitte Ihren A try: from modules.shared.auditLogger import audit_logger audit_logger.logSecurityEvent( - userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + userId=str(context.user.id), + mandateId=str(context.mandateId) if context.mandateId else "system", action="send_password_link", details=f"Sent password setup link to user {userId} ({targetUser.email})" ) except Exception: pass - logger.info(f"Password setup link sent to {targetUser.email} for user {targetUser.username} by admin {currentUser.username}") + logger.info(f"Password setup link sent to {targetUser.email} for user {targetUser.username} by admin {context.user.username}") return { "message": f"Password setup link sent to {targetUser.email}", @@ -470,10 +602,13 @@ Falls Sie diese Anforderung nicht erwartet haben, kontaktieren Sie bitte Ihren A async def delete_user( request: Request, userId: str = Path(..., description="ID of the user to delete"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Delete a user""" - appInterface = interfaceDbAppObjects.getInterface(currentUser) + """ + Delete a user. + MULTI-TENANT: Can only delete users in the same mandate (unless SysAdmin). + """ + appInterface = interfaceDbAppObjects.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) @@ -483,6 +618,25 @@ async def delete_user( detail=f"User with ID {userId} not found" ) + # MULTI-TENANT: Verify user is in the same mandate (unless SysAdmin) + if context.mandateId and not context.isSysAdmin: + from modules.datamodels.datamodelMembership import UserMandate + userMandate = appInterface.db.getRecordset(UserMandate, recordFilter={ + "userId": userId, + "mandateId": str(context.mandateId) + }) + if not userMandate: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Cannot delete user outside your mandate" + ) + + # Delete UserMandate entries for this user first + from modules.datamodels.datamodelMembership import UserMandate + userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"userId": userId}) + for um in userMandates: + appInterface.db.deleteRecord(UserMandate, um["id"]) + success = appInterface.deleteUser(userId) if not success: raise HTTPException( diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeFeatureAutomation.py similarity index 85% rename from modules/routes/routeAdminAutomationEvents.py rename to modules/routes/routeFeatureAutomation.py index a24c7839..ea58e271 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeFeatureAutomation.py @@ -12,8 +12,7 @@ import logging # Import interfaces and models import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects -from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelUam import User +from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext # Configure logger logger = logging.getLogger(__name__) @@ -31,26 +30,16 @@ router = APIRouter( } ) -def requireSysadmin(currentUser: User): - """Require sysadmin role""" - if "sysadmin" not in (currentUser.roleLabels or []): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Sysadmin role required" - ) - @router.get("") @limiter.limit("30/minute") async def get_all_automation_events( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get all automation events across all mandates (sysadmin only). Returns list of all registered events with their automation IDs and schedules. """ - requireSysadmin(currentUser) - try: from modules.shared.eventManagement import eventManager @@ -79,20 +68,18 @@ async def get_all_automation_events( @limiter.limit("5/minute") async def sync_all_automation_events( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Manually trigger sync for all automations (sysadmin only). This will register/remove events based on active flags. """ - requireSysadmin(currentUser) - try: from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.features.workflow import syncAutomationEvents - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user) # Get event user for sync operation (routes can import from interfaces) rootInterface = getRootInterface() eventUser = rootInterface.getUserByUsername("event") @@ -103,7 +90,7 @@ async def sync_all_automation_events( ) from modules.services import getInterface as getServices - services = getServices(currentUser, None) + services = getServices(context.user, None) result = await syncAutomationEvents(services, eventUser) return { "success": True, @@ -124,14 +111,12 @@ async def sync_all_automation_events( async def remove_event( request: Request, eventId: str = Path(..., description="Event ID to remove"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Manually remove a specific event from scheduler (sysadmin only). Used for debugging and manual event cleanup. """ - requireSysadmin(currentUser) - try: from modules.shared.eventManagement import eventManager @@ -141,9 +126,9 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatObjects.getInterface(currentUser) + chatInterface = interfaceDbChatObjects.getInterface(context.user) automation = chatInterface.getAutomationDefinition(automation_id) - if automation and automation.get("eventId") == eventId: + if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) return { @@ -157,4 +142,3 @@ async def remove_event( status_code=500, detail=f"Error removing event: {str(e)}" ) - diff --git a/modules/routes/routeChatPlayground.py b/modules/routes/routeFeatureChatDynamic.py similarity index 86% rename from modules/routes/routeChatPlayground.py rename to modules/routes/routeFeatureChatDynamic.py index 287543c2..f2955b61 100644 --- a/modules/routes/routeChatPlayground.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -10,14 +10,13 @@ from typing import Optional, Dict, Any from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Request # Import auth modules -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User # Import workflow control functions from modules.features.workflow import chatStart, chatStop @@ -32,8 +31,8 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) -def getServiceChat(currentUser: User): - return interfaceDbChatObjects.getInterface(currentUser) +def _getServiceChat(context: RequestContext): + return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Workflow start endpoint @router.post("/start", response_model=ChatWorkflow) @@ -43,7 +42,7 @@ async def start_workflow( workflowId: Optional[str] = Query(None, description="Optional ID of the workflow to continue"), workflowMode: WorkflowModeEnum = Query(..., description="Workflow mode: 'Dynamic' or 'Automation' (mandatory)"), userInput: UserInputRequest = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> ChatWorkflow: """ Starts a new workflow or continues an existing one. @@ -54,7 +53,7 @@ async def start_workflow( """ try: # Start or continue workflow using playground controller - workflow = await chatStart(currentUser, userInput, workflowMode, workflowId) + workflow = await chatStart(context.user, userInput, workflowMode, workflowId) return workflow @@ -71,12 +70,12 @@ async def start_workflow( async def stop_workflow( request: Request, workflowId: str = Path(..., description="ID of the workflow to stop"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> ChatWorkflow: """Stops a running workflow.""" try: # Stop workflow using playground controller - workflow = await chatStop(currentUser, workflowId) + workflow = await chatStop(context.user, workflowId) return workflow @@ -94,7 +93,7 @@ async def get_workflow_chat_data( request: Request, workflowId: str = Path(..., description="ID of the workflow"), afterTimestamp: Optional[float] = Query(None, description="Unix timestamp to get data after"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Get unified chat data (messages, logs, stats) for a workflow with timestamp-based selective data transfer. @@ -102,7 +101,7 @@ async def get_workflow_chat_data( """ try: # Get service center - interfaceDbChat = getServiceChat(currentUser) + interfaceDbChat = _getServiceChat(context) # Verify workflow exists workflow = interfaceDbChat.getWorkflow(workflowId) diff --git a/modules/routes/routeChatbot.py b/modules/routes/routeFeatureChatbot.py similarity index 96% rename from modules/routes/routeChatbot.py rename to modules/routes/routeFeatureChatbot.py index 11814313..0505a752 100644 --- a/modules/routes/routeChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -15,7 +15,7 @@ from fastapi.responses import StreamingResponse from modules.shared.timeUtils import parseTimestamp # Import auth modules -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects @@ -23,7 +23,6 @@ from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum -from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature @@ -43,8 +42,8 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) -def getServiceChat(currentUser: User): - return interfaceDbChatObjects.getInterface(currentUser) +def _getServiceChat(context: RequestContext): + return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Chatbot streaming endpoint (SSE) @router.post("/start/stream") @@ -53,7 +52,7 @@ async def stream_chatbot_start( request: Request, workflowId: Optional[str] = Query(None, description="Optional ID of the workflow to continue (can also be in request body)"), userInput: UserInputRequest = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> StreamingResponse: """ Starts a new chatbot workflow or continues an existing one with SSE streaming. @@ -71,7 +70,7 @@ async def stream_chatbot_start( final_workflow_id = workflowId or userInput.workflowId # Start background processing (this will create the workflow and event queue) - workflow = await chatProcess(currentUser, userInput, final_workflow_id) + workflow = await chatProcess(context.user, str(context.mandateId), userInput, final_workflow_id) # Get event queue for the workflow queue = event_manager.get_queue(workflow.id) @@ -83,7 +82,7 @@ async def stream_chatbot_start( """Async generator for SSE events - pure event-driven streaming (no polling).""" try: # Get interface for initial data and status checks - interfaceDbChat = getServiceChat(currentUser) + interfaceDbChat = _getServiceChat(context) # Get current workflow to check if resuming and get current round current_workflow = interfaceDbChat.getWorkflow(workflow.id) @@ -239,11 +238,11 @@ async def stream_chatbot_start( async def stop_chatbot( request: Request, workflowId: str = Path(..., description="ID of the workflow to stop"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> ChatWorkflow: """Stops a running chatbot workflow.""" try: - workflow = await chatStop(currentUser, workflowId) + workflow = await chatStop(context.user, workflowId) # Emit stopped event to active streams event_manager = get_event_manager() @@ -272,18 +271,18 @@ async def stop_chatbot( async def delete_chatbot( request: Request, workflowId: str = Path(..., description="ID of the workflow to delete"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Deletes a chatbot workflow and its associated data.""" try: # Get service center - interfaceDbChat = getServiceChat(currentUser) + interfaceDbChat = _getServiceChat(context) # Check workflow access and permission using RBAC workflows = getRecordsetWithRBAC( interfaceDbChat.db, ChatWorkflow, - currentUser, + context.user, recordFilter={"id": workflowId} ) if not workflows: @@ -337,7 +336,7 @@ async def get_chatbot_threads( request: Request, workflowId: Optional[str] = Query(None, description="Optional workflow ID to get details and chat data for a specific thread"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object (only used when workflowId is not provided)"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Union[PaginatedResponse[ChatWorkflow], Dict[str, Any]]: """ List all chatbot workflows (threads) for the current user, or get details and chat data for a specific thread. @@ -346,7 +345,7 @@ async def get_chatbot_threads( - If workflowId is not provided: Returns a paginated list of all workflows """ try: - interfaceDbChat = getServiceChat(currentUser) + interfaceDbChat = _getServiceChat(context) # If workflowId is provided, return single workflow with chat data if workflowId: @@ -456,4 +455,3 @@ async def get_chatbot_threads( status_code=500, detail=f"Error getting chatbot threads: {str(e)}" ) - diff --git a/modules/routes/routeDataNeutralization.py b/modules/routes/routeFeatureNeutralization.py similarity index 85% rename from modules/routes/routeDataNeutralization.py rename to modules/routes/routeFeatureNeutralization.py index 7826d96c..04d034dc 100644 --- a/modules/routes/routeDataNeutralization.py +++ b/modules/routes/routeFeatureNeutralization.py @@ -5,10 +5,9 @@ from typing import List, Dict, Any, Optional import logging # Import auth module -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes from modules.features.neutralizePlayground.mainNeutralizePlayground import NeutralizationPlayground @@ -32,18 +31,18 @@ router = APIRouter( @limiter.limit("30/minute") async def get_neutralization_config( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> DataNeutraliserConfig: """Get data neutralization configuration""" try: - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) config = service.getConfig() if not config: # Return default config instead of 404 return DataNeutraliserConfig( - mandateId=currentUser.mandateId, - userId=currentUser.id, + mandateId=context.mandateId, + userId=context.user.id, enabled=True, namesToParse="", sharepointSourcePath="", @@ -66,11 +65,11 @@ async def get_neutralization_config( async def save_neutralization_config( request: Request, config_data: Dict[str, Any] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> DataNeutraliserConfig: """Save or update data neutralization configuration""" try: - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) config = service.saveConfig(config_data) return config @@ -87,7 +86,7 @@ async def save_neutralization_config( async def neutralize_text( request: Request, text_data: Dict[str, Any] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Neutralize text content""" try: @@ -100,7 +99,7 @@ async def neutralize_text( detail="Text content is required" ) - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) result = service.neutralizeText(text, file_id) return result @@ -119,7 +118,7 @@ async def neutralize_text( async def resolve_text( request: Request, text_data: Dict[str, str] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, str]: """Resolve UIDs in neutralized text back to original text""" try: @@ -131,7 +130,7 @@ async def resolve_text( detail="Text content is required" ) - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) resolved_text = service.resolveText(text) return {"resolved_text": resolved_text} @@ -150,11 +149,11 @@ async def resolve_text( async def get_neutralization_attributes( request: Request, fileId: Optional[str] = Query(None, description="Filter by file ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[DataNeutralizerAttributes]: """Get neutralization attributes, optionally filtered by file ID""" try: - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) attributes = service.getAttributes(fileId) return attributes @@ -171,7 +170,7 @@ async def get_neutralization_attributes( async def process_sharepoint_files( request: Request, paths_data: Dict[str, str] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Process files from SharePoint source path and store neutralized files in target path""" try: @@ -184,7 +183,7 @@ async def process_sharepoint_files( detail="Both source and target paths are required" ) - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) result = await service.processSharepointFiles(source_path, target_path) return result @@ -203,7 +202,7 @@ async def process_sharepoint_files( async def batch_process_files( request: Request, files_data: List[Dict[str, Any]] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Process multiple files for neutralization""" try: @@ -213,7 +212,7 @@ async def batch_process_files( detail="Files data is required" ) - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) result = service.batchNeutralizeFiles(files_data) return result @@ -231,11 +230,11 @@ async def batch_process_files( @limiter.limit("30/minute") async def get_neutralization_stats( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Get neutralization processing statistics""" try: - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) stats = service.getProcessingStats() return stats @@ -252,11 +251,11 @@ async def get_neutralization_stats( async def cleanup_file_attributes( request: Request, fileId: str = Path(..., description="File ID to cleanup attributes for"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, str]: """Clean up neutralization attributes for a specific file""" try: - service = NeutralizationPlayground(currentUser) + service = NeutralizationPlayground(context.user, str(context.mandateId)) success = service.cleanupFileAttributes(fileId) if success: diff --git a/modules/routes/routeRealEstate.py b/modules/routes/routeFeatureRealEstate.py similarity index 91% rename from modules/routes/routeRealEstate.py rename to modules/routes/routeFeatureRealEstate.py index a554ce7d..fe7544de 100644 --- a/modules/routes/routeRealEstate.py +++ b/modules/routes/routeFeatureRealEstate.py @@ -10,10 +10,9 @@ from typing import Optional, Dict, Any, List, Union from fastapi import APIRouter, HTTPException, Depends, Body, Request, Query, Path, status # Import auth modules -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getRequestContext, RequestContext # Import models -from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata from modules.datamodels.datamodelRealEstate import ( Projekt, @@ -63,7 +62,7 @@ router = APIRouter( async def process_command( request: Request, userInput: str = Body(..., embed=True, description="Natural language command"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Process natural language command and execute corresponding CRUD operation. @@ -73,9 +72,9 @@ async def process_command( Example user inputs: - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - - "Zeige mir alle Projekte in Zürich" + - "Zeige mir alle Projekte in Zuerich" - "Aktualisiere Projekt XYZ mit Status 'Planung'" - - "Lösche Parzelle ABC" + - "Loesche Parzelle ABC" - "SELECT * FROM Projekt WHERE plz = '8000'" Headers: @@ -93,7 +92,7 @@ async def process_command( # Validate CSRF token (middleware also checks, but explicit validation for better error messages) csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/command from user {currentUser.id}") + logger.warning(f"CSRF token missing for POST /api/realestate/command from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." @@ -101,7 +100,7 @@ async def process_command( # Basic CSRF token format validation if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for POST /api/realestate/command from user {currentUser.id}") + logger.warning(f"Invalid CSRF token format for POST /api/realestate/command from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" @@ -111,18 +110,19 @@ async def process_command( try: int(csrf_token, 16) except ValueError: - logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/command from user {currentUser.id}") + logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/command from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" ) - logger.info(f"Processing command request from user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Processing command request from user {context.user.id} (mandate: {context.mandateId})") logger.debug(f"User input: {userInput}") # Process natural language command with AI result = await processNaturalLanguageCommand( - currentUser=currentUser, + currentUser=context.user, + mandateId=str(context.mandateId), userInput=userInput ) @@ -147,7 +147,7 @@ async def process_command( @limiter.limit("120/minute") async def get_available_tables( request: Request, - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Get all available real estate tables. @@ -164,7 +164,7 @@ async def get_available_tables( # Validate CSRF token if provided csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/tables from user {currentUser.id}") + logger.warning(f"CSRF token missing for GET /api/realestate/tables from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." @@ -172,7 +172,7 @@ async def get_available_tables( # Basic CSRF token format validation if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for GET /api/realestate/tables from user {currentUser.id}") + logger.warning(f"Invalid CSRF token format for GET /api/realestate/tables from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" @@ -182,13 +182,13 @@ async def get_available_tables( try: int(csrf_token, 16) except ValueError: - logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/tables from user {currentUser.id}") + logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/tables from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" ) - logger.info(f"Getting available tables for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Getting available tables for user {context.user.id} (mandate: {context.mandateId})") # Define available tables with descriptions tables = [ @@ -245,7 +245,7 @@ async def get_table_data( request: Request, table: str = Path(..., description="Table name (Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land)"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[Dict[str, Any]]: """ Get all data from a specific real estate table with optional pagination. @@ -273,7 +273,7 @@ async def get_table_data( # Validate CSRF token if provided csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"CSRF token missing for GET /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." @@ -281,7 +281,7 @@ async def get_table_data( # Basic CSRF token format validation if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for GET /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"Invalid CSRF token format for GET /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" @@ -291,13 +291,13 @@ async def get_table_data( try: int(csrf_token, 16) except ValueError: - logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" ) - logger.info(f"Getting table data for '{table}' from user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Getting table data for '{table}' from user {context.user.id} (mandate: {context.mandateId})") # Map table names to model classes and getter methods table_mapping = { @@ -317,7 +317,7 @@ async def get_table_data( ) # Get interface and fetch data - realEstateInterface = getRealEstateInterface(currentUser) + realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) model_class, method_name = table_mapping[table] getter_method = getattr(realEstateInterface, method_name) @@ -399,7 +399,7 @@ async def create_table_record( request: Request, table: str = Path(..., description="Table name (Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land)"), data: Dict[str, Any] = Body(..., description="Record data to create"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Create a new record in a specific real estate table. @@ -442,7 +442,7 @@ async def create_table_record( # Validate CSRF token csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"CSRF token missing for POST /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." @@ -450,7 +450,7 @@ async def create_table_record( # Basic CSRF token format validation if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for POST /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"Invalid CSRF token format for POST /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" @@ -460,7 +460,7 @@ async def create_table_record( try: int(csrf_token, 16) except ValueError: - logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/table/{table} from user {currentUser.id}") + logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/table/{table} from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Invalid CSRF token format" @@ -468,7 +468,7 @@ async def create_table_record( # Special handling for Projekt with parcel data if table == "Projekt" and ("parzelle" in data or "parzellen" in data): - logger.info(f"Creating Projekt with parcel data for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Creating Projekt with parcel data for user {context.user.id} (mandate: {context.mandateId})") # Extract fields label = data.get("label") @@ -491,7 +491,7 @@ async def create_table_record( detail="parzellen must be an array" ) elif "parzelle" in data: - # Single parcel (backward compatibility) + # Single parcel parzelle_data = data.get("parzelle") if parzelle_data: parzellen_data = [parzelle_data] @@ -505,7 +505,8 @@ async def create_table_record( # Use helper function to create project with parcel data try: result = await create_project_with_parcel_data( - currentUser=currentUser, + currentUser=context.user, + mandateId=str(context.mandateId), projekt_label=label, parzellen_data=parzellen_data, status_prozess=status_prozess, @@ -524,7 +525,7 @@ async def create_table_record( ) # Standard handling for other tables or Projekt without parcel data - logger.info(f"Creating record in table '{table}' for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Creating record in table '{table}' for user {context.user.id} (mandate: {context.mandateId})") logger.debug(f"Record data: {data}") # Map table names to model classes and create methods @@ -545,13 +546,13 @@ async def create_table_record( ) # Get interface - realEstateInterface = getRealEstateInterface(currentUser) + realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) model_class, method_name = table_mapping[table] create_method = getattr(realEstateInterface, method_name) - # Ensure mandateId is set (will be set by interface if missing) + # Ensure mandateId is set from context if "mandateId" not in data: - data["mandateId"] = currentUser.mandateId + data["mandateId"] = str(context.mandateId) if context.mandateId else None # Create model instance from data try: @@ -596,7 +597,7 @@ async def search_parcel( request: Request, location: str = Query(..., description="Either coordinates as 'x,y' (LV95) or address string"), include_adjacent: bool = Query(False, description="Include adjacent parcels information"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Search for parcel information by address or coordinates. @@ -614,50 +615,18 @@ async def search_parcel( Headers: - X-CSRF-Token: CSRF token (required for security) - - Examples: - - GET /api/realestate/parcel/search?location=2600000,1200000 - - GET /api/realestate/parcel/search?location=Bundesplatz 3, 3003 Bern - - GET /api/realestate/parcel/search?location=Bundesplatz 3, 3003 Bern&include_adjacent=true - - Returns: - { - "parcel": { - "id": "823", - "egrid": "CH294676423526", - "number": "823", - "name": "823", - "identnd": "BE0200000042", - "canton": "BE", - "municipality_code": 351, - "municipality_name": "Bern", - "address": "Bundesplatz 3 3011 Bern", - "plz": "3011", - "perimeter": {...}, - "area_m2": 1234.56, - "centroid": {"x": 2600000, "y": 1200000}, - "geoportal_url": "https://...", - "realestate_type": null - }, - "map_view": { - "center": {"x": 2600000, "y": 1200000}, - "zoom_bounds": {"min_x": ..., "max_x": ..., "min_y": ..., "max_y": ...}, - "geometry_geojson": {...} - }, - "adjacent_parcels": [...] // Optional (only if include_adjacent=true) - } """ try: # Validate CSRF token csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/parcel/search from user {currentUser.id}") + logger.warning(f"CSRF token missing for GET /api/realestate/parcel/search from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." ) - logger.info(f"Searching parcel for user {currentUser.id} (mandate: {currentUser.mandateId}) with location: {location}") + logger.info(f"Searching parcel for user {context.user.id} (mandate: {context.mandateId}) with location: {location}") # Initialize connector connector = SwissTopoMapServerConnector() @@ -762,15 +731,14 @@ async def search_parcel( # Basic municipality lookup for common codes common_municipalities = { 351: "Bern", - 261: "Zürich", - 6621: "Genève", + 261: "Zuerich", + 6621: "Geneve", 2701: "Basel", 5586: "Lausanne", 1061: "Luzern", 3203: "Winterthur", 230: "St. Gallen", 5192: "Lugano", - 351: "Bern", 1367: "Schwyz" } @@ -944,7 +912,7 @@ async def add_parcel_to_project( request: Request, projekt_id: str = Path(..., description="Projekt ID"), body: Dict[str, Any] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Add a parcel to an existing project. @@ -961,7 +929,7 @@ async def add_parcel_to_project( Option 2 - Create new parcel from location: { - "location": "Hauptstrasse 42, 8000 Zürich" + "location": "Hauptstrasse 42, 8000 Zuerich" } Option 3 - Create new parcel with custom data: @@ -988,7 +956,7 @@ async def add_parcel_to_project( # Validate CSRF token csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/projekt/{projekt_id}/add-parcel from user {currentUser.id}") + logger.warning(f"CSRF token missing for POST /api/realestate/projekt/{projekt_id}/add-parcel from user {context.user.id}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="CSRF token missing. Please include X-CSRF-Token header." @@ -1008,15 +976,16 @@ async def add_parcel_to_project( detail="Invalid CSRF token format" ) - logger.info(f"Adding parcel to project {projekt_id} for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.info(f"Adding parcel to project {projekt_id} for user {context.user.id} (mandate: {context.mandateId})") # Get interface - realEstateInterface = getRealEstateInterface(currentUser) + realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - # Fetch existing Projekt - projekte = realEstateInterface.getProjekte( - recordFilter={"id": projekt_id, "mandateId": currentUser.mandateId} - ) + # Fetch existing Projekt - use mandateId from context + recordFilter = {"id": projekt_id} + if context.mandateId: + recordFilter["mandateId"] = str(context.mandateId) + projekte = realEstateInterface.getProjekte(recordFilter=recordFilter) if not projekte: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -1034,9 +1003,10 @@ async def add_parcel_to_project( # Option 1: Link existing parcel if parcel_id: logger.info(f"Linking existing parcel {parcel_id}") - parcels = realEstateInterface.getParzellen( - recordFilter={"id": parcel_id, "mandateId": currentUser.mandateId} - ) + parcelFilter = {"id": parcel_id} + if context.mandateId: + parcelFilter["mandateId"] = str(context.mandateId) + parcels = realEstateInterface.getParzellen(recordFilter=parcelFilter) if not parcels: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -1062,9 +1032,9 @@ async def add_parcel_to_project( extracted_attributes = connector.extract_parcel_attributes(parcel_data) attributes = parcel_data.get("attributes", {}) - # Create Parzelle + # Create Parzelle with mandateId from context parzelle_create_data = { - "mandateId": currentUser.mandateId, + "mandateId": str(context.mandateId) if context.mandateId else None, "label": extracted_attributes.get("label") or attributes.get("number") or "Unknown", "parzellenAliasTags": [attributes.get("egris_egrid")] if attributes.get("egris_egrid") else [], "eigentuemerschaft": None, @@ -1111,7 +1081,7 @@ async def add_parcel_to_project( # Option 3: Create from custom data elif parcel_data_dict: logger.info(f"Creating parcel from custom data") - parcel_data_dict["mandateId"] = currentUser.mandateId + parcel_data_dict["mandateId"] = str(context.mandateId) if context.mandateId else None parzelle_instance = Parzelle(**parcel_data_dict) parzelle = realEstateInterface.createParzelle(parzelle_instance) @@ -1150,4 +1120,3 @@ async def add_parcel_to_project( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error adding parcel to project: {str(e)}" ) - diff --git a/modules/routes/routeDataTrustee.py b/modules/routes/routeFeatureTrustee.py similarity index 79% rename from modules/routes/routeDataTrustee.py rename to modules/routes/routeFeatureTrustee.py index bca55df4..69fd5918 100644 --- a/modules/routes/routeDataTrustee.py +++ b/modules/routes/routeFeatureTrustee.py @@ -13,7 +13,7 @@ import logging import json import io -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getRequestContext, RequestContext from modules.interfaces.interfaceDbTrusteeObjects import getInterface from modules.datamodels.datamodelTrustee import ( TrusteeOrganisation, @@ -24,7 +24,6 @@ from modules.datamodels.datamodelTrustee import ( TrusteePosition, TrusteePositionDocument, ) -from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelPagination import ( PaginationParams, PaginatedResponse, @@ -67,13 +66,13 @@ def _parsePagination(pagination: Optional[str]) -> Optional[PaginationParams]: async def getOrganisations( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeOrganisation]: """Get all organisations with optional pagination.""" logger = logging.getLogger(__name__) - logger.debug(f"getOrganisations called for user {currentUser.id}, roles: {currentUser.roleLabels}") + logger.debug(f"getOrganisations called for user {context.user.id}, mandateId: {context.mandateId}") paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllOrganisations(paginationParams) logger.debug(f"getOrganisations returned {len(result.items)} items") @@ -97,14 +96,14 @@ async def getOrganisations( async def getOrganisation( request: Request, orgId: str = Path(..., description="Organisation ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Get a single organisation by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) org = interface.getOrganisation(orgId) if not org: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") - return TrusteeOrganisation(**org) + return org @router.post("/organisations", response_model=TrusteeOrganisation, status_code=201) @@ -112,14 +111,14 @@ async def getOrganisation( async def createOrganisation( request: Request, data: TrusteeOrganisation = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Create a new organisation.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createOrganisation(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create organisation") - return TrusteeOrganisation(**result) + return result @router.put("/organisations/{orgId}", response_model=TrusteeOrganisation) @@ -128,10 +127,10 @@ async def updateOrganisation( request: Request, orgId: str = Path(..., description="Organisation ID"), data: TrusteeOrganisation = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Update an organisation.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -139,7 +138,7 @@ async def updateOrganisation( result = interface.updateOrganisation(orgId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update organisation") - return TrusteeOrganisation(**result) + return result @router.delete("/organisations/{orgId}") @@ -147,10 +146,10 @@ async def updateOrganisation( async def deleteOrganisation( request: Request, orgId: str = Path(..., description="Organisation ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an organisation.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -168,11 +167,11 @@ async def deleteOrganisation( async def getRoles( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeRole]: """Get all roles with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllRoles(paginationParams) if paginationParams: @@ -195,14 +194,14 @@ async def getRoles( async def getRole( request: Request, roleId: str = Path(..., description="Role ID"), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Get a single role by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) role = interface.getRole(roleId) if not role: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") - return TrusteeRole(**role) + return role @router.post("/roles", response_model=TrusteeRole, status_code=201) @@ -210,14 +209,14 @@ async def getRole( async def createRole( request: Request, data: TrusteeRole = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Create a new role (sysadmin only).""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createRole(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create role") - return TrusteeRole(**result) + return result @router.put("/roles/{roleId}", response_model=TrusteeRole) @@ -226,10 +225,10 @@ async def updateRole( request: Request, roleId: str = Path(...), data: TrusteeRole = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Update a role (sysadmin only).""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -237,7 +236,7 @@ async def updateRole( result = interface.updateRole(roleId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update role") - return TrusteeRole(**result) + return result @router.delete("/roles/{roleId}") @@ -245,10 +244,10 @@ async def updateRole( async def deleteRole( request: Request, roleId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a role (sysadmin only, fails if in use).""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -266,11 +265,11 @@ async def deleteRole( async def getAllAccess( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeAccess]: """Get all access records with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllAccess(paginationParams) if paginationParams: @@ -293,14 +292,14 @@ async def getAllAccess( async def getAccess( request: Request, accessId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Get a single access record by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) access = interface.getAccess(accessId) if not access: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") - return TrusteeAccess(**access) + return access @router.get("/access/organisation/{orgId}", response_model=List[TrusteeAccess]) @@ -308,11 +307,11 @@ async def getAccess( async def getAccessByOrganisation( request: Request, orgId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for an organisation.""" - interface = getInterface(currentUser) - return [TrusteeAccess(**a) for a in interface.getAccessByOrganisation(orgId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getAccessByOrganisation(orgId) @router.get("/access/user/{userId}", response_model=List[TrusteeAccess]) @@ -320,11 +319,11 @@ async def getAccessByOrganisation( async def getAccessByUser( request: Request, userId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for a user.""" - interface = getInterface(currentUser) - return [TrusteeAccess(**a) for a in interface.getAccessByUser(userId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getAccessByUser(userId) @router.post("/access", response_model=TrusteeAccess, status_code=201) @@ -332,14 +331,14 @@ async def getAccessByUser( async def createAccess( request: Request, data: TrusteeAccess = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Create a new access record.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createAccess(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create access") - return TrusteeAccess(**result) + return result @router.put("/access/{accessId}", response_model=TrusteeAccess) @@ -348,10 +347,10 @@ async def updateAccess( request: Request, accessId: str = Path(...), data: TrusteeAccess = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Update an access record.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -359,7 +358,7 @@ async def updateAccess( result = interface.updateAccess(accessId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update access") - return TrusteeAccess(**result) + return result @router.delete("/access/{accessId}") @@ -367,10 +366,10 @@ async def updateAccess( async def deleteAccess( request: Request, accessId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an access record.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -388,11 +387,11 @@ async def deleteAccess( async def getContracts( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeContract]: """Get all contracts with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllContracts(paginationParams) if paginationParams: @@ -415,14 +414,14 @@ async def getContracts( async def getContract( request: Request, contractId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Get a single contract by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) contract = interface.getContract(contractId) if not contract: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") - return TrusteeContract(**contract) + return contract @router.get("/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) @@ -430,11 +429,11 @@ async def getContract( async def getContractsByOrganisation( request: Request, orgId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeContract]: """Get all contracts for an organisation.""" - interface = getInterface(currentUser) - return [TrusteeContract(**c) for c in interface.getContractsByOrganisation(orgId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getContractsByOrganisation(orgId) @router.post("/contracts", response_model=TrusteeContract, status_code=201) @@ -442,14 +441,14 @@ async def getContractsByOrganisation( async def createContract( request: Request, data: TrusteeContract = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Create a new contract.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createContract(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create contract") - return TrusteeContract(**result) + return result @router.put("/contracts/{contractId}", response_model=TrusteeContract) @@ -458,10 +457,10 @@ async def updateContract( request: Request, contractId: str = Path(...), data: TrusteeContract = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Update a contract (organisationId is immutable).""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -469,7 +468,7 @@ async def updateContract( result = interface.updateContract(contractId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update contract (organisationId cannot be changed)") - return TrusteeContract(**result) + return result @router.delete("/contracts/{contractId}") @@ -477,10 +476,10 @@ async def updateContract( async def deleteContract( request: Request, contractId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a contract.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -498,11 +497,11 @@ async def deleteContract( async def getDocuments( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeDocument]: """Get all documents (metadata only) with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllDocuments(paginationParams) if paginationParams: @@ -525,14 +524,14 @@ async def getDocuments( async def getDocument( request: Request, documentId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Get document metadata by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") - return TrusteeDocument(**doc) + return doc @router.get("/documents/{documentId}/data") @@ -540,10 +539,10 @@ async def getDocument( async def getDocumentData( request: Request, documentId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ): """Download document binary data.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -554,8 +553,8 @@ async def getDocumentData( return StreamingResponse( io.BytesIO(data), - media_type=doc.get("documentMimeType", "application/octet-stream"), - headers={"Content-Disposition": f"attachment; filename={doc.get('documentName', 'document')}"} + media_type=doc.documentMimeType or "application/octet-stream", + headers={"Content-Disposition": f"attachment; filename={doc.documentName or 'document'}"} ) @@ -564,11 +563,11 @@ async def getDocumentData( async def getDocumentsByContract( request: Request, contractId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeDocument]: """Get all documents for a contract.""" - interface = getInterface(currentUser) - return [TrusteeDocument(**d) for d in interface.getDocumentsByContract(contractId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getDocumentsByContract(contractId) @router.post("/documents", response_model=TrusteeDocument, status_code=201) @@ -576,14 +575,14 @@ async def getDocumentsByContract( async def createDocument( request: Request, data: TrusteeDocument = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Create a new document.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create document") - return TrusteeDocument(**result) + return result @router.put("/documents/{documentId}", response_model=TrusteeDocument) @@ -592,10 +591,10 @@ async def updateDocument( request: Request, documentId: str = Path(...), data: TrusteeDocument = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Update document metadata.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -603,7 +602,7 @@ async def updateDocument( result = interface.updateDocument(documentId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update document") - return TrusteeDocument(**result) + return result @router.delete("/documents/{documentId}") @@ -611,10 +610,10 @@ async def updateDocument( async def deleteDocument( request: Request, documentId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a document.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -632,11 +631,11 @@ async def deleteDocument( async def getPositions( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePosition]: """Get all positions with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllPositions(paginationParams) if paginationParams: @@ -659,14 +658,14 @@ async def getPositions( async def getPosition( request: Request, positionId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Get a single position by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) position = interface.getPosition(positionId) if not position: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") - return TrusteePosition(**position) + return position @router.get("/positions/contract/{contractId}", response_model=List[TrusteePosition]) @@ -674,11 +673,11 @@ async def getPosition( async def getPositionsByContract( request: Request, contractId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for a contract.""" - interface = getInterface(currentUser) - return [TrusteePosition(**p) for p in interface.getPositionsByContract(contractId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getPositionsByContract(contractId) @router.get("/positions/organisation/{orgId}", response_model=List[TrusteePosition]) @@ -686,11 +685,11 @@ async def getPositionsByContract( async def getPositionsByOrganisation( request: Request, orgId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for an organisation.""" - interface = getInterface(currentUser) - return [TrusteePosition(**p) for p in interface.getPositionsByOrganisation(orgId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getPositionsByOrganisation(orgId) @router.post("/positions", response_model=TrusteePosition, status_code=201) @@ -698,14 +697,14 @@ async def getPositionsByOrganisation( async def createPosition( request: Request, data: TrusteePosition = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Create a new position.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createPosition(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create position") - return TrusteePosition(**result) + return result @router.put("/positions/{positionId}", response_model=TrusteePosition) @@ -714,10 +713,10 @@ async def updatePosition( request: Request, positionId: str = Path(...), data: TrusteePosition = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Update a position.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -725,7 +724,7 @@ async def updatePosition( result = interface.updatePosition(positionId, data.model_dump(exclude={"id"})) if not result: raise HTTPException(status_code=400, detail="Failed to update position") - return TrusteePosition(**result) + return result @router.delete("/positions/{positionId}") @@ -733,10 +732,10 @@ async def updatePosition( async def deletePosition( request: Request, positionId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -754,11 +753,11 @@ async def deletePosition( async def getPositionDocuments( request: Request, pagination: Optional[str] = Query(None), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePositionDocument]: """Get all position-document links with optional pagination.""" paginationParams = _parsePagination(pagination) - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.getAllPositionDocuments(paginationParams) if paginationParams: @@ -781,14 +780,14 @@ async def getPositionDocuments( async def getPositionDocument( request: Request, linkId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Get a single position-document link by ID.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) link = interface.getPositionDocument(linkId) if not link: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") - return TrusteePositionDocument(**link) + return link @router.get("/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) @@ -796,11 +795,11 @@ async def getPositionDocument( async def getDocumentsForPosition( request: Request, positionId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all document links for a position.""" - interface = getInterface(currentUser) - return [TrusteePositionDocument(**l) for l in interface.getDocumentsForPosition(positionId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getDocumentsForPosition(positionId) @router.get("/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) @@ -808,11 +807,11 @@ async def getDocumentsForPosition( async def getPositionsForDocument( request: Request, documentId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all position links for a document.""" - interface = getInterface(currentUser) - return [TrusteePositionDocument(**l) for l in interface.getPositionsForDocument(documentId)] + interface = getInterface(context.user, mandateId=context.mandateId) + return interface.getPositionsForDocument(documentId) @router.post("/position-documents", response_model=TrusteePositionDocument, status_code=201) @@ -820,14 +819,14 @@ async def getPositionsForDocument( async def createPositionDocument( request: Request, data: TrusteePositionDocument = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Create a new position-document link.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) result = interface.createPositionDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create link") - return TrusteePositionDocument(**result) + return result @router.delete("/position-documents/{linkId}") @@ -835,10 +834,10 @@ async def createPositionDocument( async def deletePositionDocument( request: Request, linkId: str = Path(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position-document link.""" - interface = getInterface(currentUser) + interface = getInterface(context.user, mandateId=context.mandateId) existing = interface.getPositionDocument(linkId) if not existing: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py new file mode 100644 index 00000000..4d724218 --- /dev/null +++ b/modules/routes/routeFeatures.py @@ -0,0 +1,625 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Feature management routes for the backend API. +Implements endpoints for Feature and FeatureInstance management. + +Multi-Tenant Design: +- Feature definitions are global (SysAdmin can manage) +- FeatureInstances belong to mandates (Mandate Admin can manage) +- Template roles are copied on instance creation +""" + +from fastapi import APIRouter, HTTPException, Depends, Request, Query +from typing import List, Dict, Any, Optional +from fastapi import status +import logging +from pydantic import BaseModel, Field + +from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelFeatures import Feature, FeatureInstance +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/features", + tags=["Features"], + responses={404: {"description": "Not found"}} +) + + +# ============================================================================= +# Request/Response Models +# ============================================================================= + +class FeatureInstanceCreate(BaseModel): + """Request model for creating a feature instance""" + featureCode: str = Field(..., description="Feature code (e.g., 'trustee', 'chatbot')") + label: str = Field(..., description="Instance label (e.g., 'Buchhaltung 2025')") + copyTemplateRoles: bool = Field(True, description="Whether to copy template roles on creation") + + +class FeatureInstanceResponse(BaseModel): + """Response model for feature instance""" + id: str + featureCode: str + mandateId: str + label: str + enabled: bool + + +class SyncRolesResult(BaseModel): + """Response model for role synchronization""" + added: int + removed: int + unchanged: int + + +# ============================================================================= +# Feature Endpoints (Global - mostly read-only for non-SysAdmin) +# ============================================================================= + +@router.get("/", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def listFeatures( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + List all available features. + + Returns global feature definitions that can be activated for mandates. + Any authenticated user can see available features. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + features = featureInterface.getAllFeatures() + return [f.model_dump() for f in features] + + except Exception as e: + logger.error(f"Error listing features: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list features: {str(e)}" + ) + + +@router.get("/{featureCode}", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getFeature( + request: Request, + featureCode: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Get a specific feature by code. + + Args: + featureCode: Feature code (e.g., 'trustee', 'chatbot') + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + feature = featureInterface.getFeature(featureCode) + if not feature: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature '{featureCode}' not found" + ) + + return feature.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting feature {featureCode}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature: {str(e)}" + ) + + +@router.post("/", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def createFeature( + request: Request, + code: str = Query(..., description="Unique feature code"), + label: Dict[str, str] = None, + icon: str = Query("mdi-puzzle", description="Icon identifier"), + sysAdmin: User = Depends(requireSysAdmin) +) -> Dict[str, Any]: + """ + Create a new feature definition. + + SysAdmin only - creates a global feature that can be activated for mandates. + + Args: + code: Unique feature code (e.g., 'trustee') + label: I18n labels (e.g., {"en": "Trustee", "de": "Treuhand"}) + icon: Icon identifier + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Check if feature already exists + existing = featureInterface.getFeature(code) + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Feature '{code}' already exists" + ) + + feature = featureInterface.createFeature( + code=code, + label=label or {"en": code.title(), "de": code.title()}, + icon=icon + ) + + logger.info(f"SysAdmin {sysAdmin.id} created feature '{code}'") + return feature.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating feature: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create feature: {str(e)}" + ) + + +# ============================================================================= +# Feature Instance Endpoints (Mandate-scoped) +# ============================================================================= + +@router.get("/instances", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def listFeatureInstances( + request: Request, + featureCode: Optional[str] = Query(None, description="Filter by feature code"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + List feature instances for the current mandate. + + Returns instances the user has access to within the selected mandate. + + Args: + featureCode: Optional filter by feature code + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instances = featureInterface.getFeatureInstancesForMandate( + mandateId=str(context.mandateId), + featureCode=featureCode + ) + + return [inst.model_dump() for inst in instances] + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing feature instances: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list feature instances: {str(e)}" + ) + + +@router.get("/instances/{instanceId}", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getFeatureInstance( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Get a specific feature instance. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access (unless SysAdmin) + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + return instance.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting feature instance {instanceId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature instance: {str(e)}" + ) + + +@router.post("/instances", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def createFeatureInstance( + request: Request, + data: FeatureInstanceCreate, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Create a new feature instance for the current mandate. + + Requires Mandate-Admin role. Template roles are optionally copied. + + Args: + data: Feature instance creation data + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to create feature instances" + ) + + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify feature exists + feature = featureInterface.getFeature(data.featureCode) + if not feature: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature '{data.featureCode}' not found" + ) + + instance = featureInterface.createFeatureInstance( + featureCode=data.featureCode, + mandateId=str(context.mandateId), + label=data.label, + copyTemplateRoles=data.copyTemplateRoles + ) + + logger.info( + f"User {context.user.id} created feature instance '{data.label}' " + f"for feature '{data.featureCode}' in mandate {context.mandateId}" + ) + + return instance.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create feature instance: {str(e)}" + ) + + +@router.delete("/instances/{instanceId}", response_model=Dict[str, str]) +@limiter.limit("10/minute") +async def deleteFeatureInstance( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, str]: + """ + Delete a feature instance. + + Requires Mandate-Admin role. CASCADE will delete associated roles and access records. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Get instance to verify access + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to delete feature instances" + ) + + featureInterface.deleteFeatureInstance(instanceId) + + logger.info(f"User {context.user.id} deleted feature instance {instanceId}") + + return {"message": "Feature instance deleted", "instanceId": instanceId} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting feature instance {instanceId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to delete feature instance: {str(e)}" + ) + + +@router.post("/instances/{instanceId}/sync-roles", response_model=SyncRolesResult) +@limiter.limit("10/minute") +async def syncInstanceRoles( + request: Request, + instanceId: str, + addOnly: bool = Query(True, description="Only add missing roles, don't remove extras"), + context: RequestContext = Depends(getRequestContext) +) -> SyncRolesResult: + """ + Synchronize roles of a feature instance with current templates. + + IMPORTANT: Templates are only copied when a FeatureInstance is created. + This sync function is for manual re-synchronization, not automatic propagation. + + Args: + instanceId: FeatureInstance ID + addOnly: If True, only add missing roles. If False, also remove extras. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Get instance to verify access + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission (Mandate-Admin or Feature-Admin) + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to sync roles" + ) + + result = featureInterface.syncRolesFromTemplate(instanceId, addOnly) + + logger.info( + f"User {context.user.id} synced roles for instance {instanceId}: {result}" + ) + + return SyncRolesResult(**result) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error syncing roles for instance {instanceId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to sync roles: {str(e)}" + ) + + +# ============================================================================= +# Template Role Endpoints (SysAdmin only) +# ============================================================================= + +@router.get("/templates/roles", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def listTemplateRoles( + request: Request, + featureCode: Optional[str] = Query(None, description="Filter by feature code"), + sysAdmin: User = Depends(requireSysAdmin) +) -> List[Dict[str, Any]]: + """ + List global template roles. + + SysAdmin only - returns template roles that are copied to new feature instances. + + Args: + featureCode: Optional filter by feature code + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + roles = featureInterface.getTemplateRoles(featureCode) + return [r.model_dump() for r in roles] + + except Exception as e: + logger.error(f"Error listing template roles: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list template roles: {str(e)}" + ) + + +@router.post("/templates/roles", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def createTemplateRole( + request: Request, + roleLabel: str = Query(..., description="Role label (e.g., 'admin', 'viewer')"), + featureCode: str = Query(..., description="Feature code this role belongs to"), + description: Dict[str, str] = None, + sysAdmin: User = Depends(requireSysAdmin) +) -> Dict[str, Any]: + """ + Create a global template role for a feature. + + SysAdmin only - new template roles are NOT automatically propagated to existing instances. + Use the sync-roles endpoint to manually synchronize. + + Args: + roleLabel: Role label + featureCode: Feature code + description: I18n descriptions + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify feature exists + feature = featureInterface.getFeature(featureCode) + if not feature: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature '{featureCode}' not found" + ) + + role = featureInterface.createTemplateRole( + roleLabel=roleLabel, + featureCode=featureCode, + description=description + ) + + logger.info( + f"SysAdmin {sysAdmin.id} created template role '{roleLabel}' " + f"for feature '{featureCode}'" + ) + + return role.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating template role: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create template role: {str(e)}" + ) + + +# ============================================================================= +# My Feature Instances (No mandate context needed) +# ============================================================================= + +@router.get("/my", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getMyFeatureInstances( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + Get all feature instances the current user has access to. + + Returns instances across all mandates the user is member of. + This endpoint does not require X-Mandate-Id header. + """ + try: + rootInterface = getRootInterface() + + # Get all feature accesses for this user + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + + if not featureAccesses: + return [] + + featureInterface = getFeatureInterface(rootInterface.db) + result = [] + + for access in featureAccesses: + if not access.enabled: + continue + + instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) + if instance and instance.enabled: + result.append({ + **instance.model_dump(), + "accessId": str(access.id) + }) + + return result + + except Exception as e: + logger.error(f"Error getting user's feature instances: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature instances: {str(e)}" + ) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _hasMandateAdminRole(context: RequestContext) -> bool: + """ + Check if the user has mandate admin role in the current context. + + A user is mandate admin if they have the 'admin' role at mandate level. + """ + if context.isSysAdmin: + return True + + if not context.roleIds: + return False + + # Check if any of the user's roles is an admin role + try: + rootInterface = getRootInterface() + from modules.datamodels.datamodelRbac import Role + + for roleId in context.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + # Admin role at mandate level (not feature-instance level) + if roleLabel == "admin" and role.get("mandateId") and not role.get("featureInstanceId"): + return True + + return False + + except Exception as e: + logger.error(f"Error checking mandate admin role: {e}") + return False diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py new file mode 100644 index 00000000..3eef1980 --- /dev/null +++ b/modules/routes/routeGdpr.py @@ -0,0 +1,514 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +GDPR compliance routes for the backend API. +Implements data subject rights according to GDPR regulations. + +GDPR Articles implemented: +- Article 15: Right of access (data export) +- Article 16: Right to rectification (via existing update endpoints) +- Article 17: Right to erasure (account deletion) +- Article 20: Right to data portability (machine-readable export) +""" + +from fastapi import APIRouter, HTTPException, Depends, Request +from fastapi.responses import JSONResponse +from typing import List, Dict, Any, Optional +from fastapi import status +import logging +import json +from pydantic import BaseModel, Field + +from modules.auth import limiter, getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared.auditLogger import audit_logger + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/user/me", + tags=["GDPR"], + responses={404: {"description": "Not found"}} +) + + +# ============================================================================= +# Response Models +# ============================================================================= + +class DataExportResponse(BaseModel): + """Response model for GDPR data export""" + exportedAt: float + userId: str + userData: Dict[str, Any] + mandates: List[Dict[str, Any]] + featureAccesses: List[Dict[str, Any]] + invitationsCreated: List[Dict[str, Any]] + invitationsUsed: List[Dict[str, Any]] + + +class DataPortabilityResponse(BaseModel): + """Machine-readable data portability response (JSON-LD format)""" + context: str = Field(alias="@context") + type: str = Field(alias="@type") + identifier: str + exportDate: str + data: Dict[str, Any] + + +class DeletionResult(BaseModel): + """Result of account deletion""" + success: bool + userId: str + deletedAt: float + deletedData: List[str] + message: str + + +# ============================================================================= +# Article 15: Right of Access +# ============================================================================= + +@router.get("/data-export", response_model=DataExportResponse) +@limiter.limit("5/minute") +async def exportUserData( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> DataExportResponse: + """ + Export all personal data (GDPR Article 15). + + Returns all data associated with the authenticated user including: + - User profile data + - Mandate memberships + - Feature access records + - Invitations created and used + + Note: This exports Gateway-level data only. Feature-specific data + (e.g., chat workflows, trustee contracts) should be exported via + feature-specific endpoints. + """ + try: + rootInterface = getRootInterface() + + # User data (exclude sensitive fields) + userData = { + "id": str(currentUser.id), + "username": currentUser.username, + "email": currentUser.email, + "firstname": currentUser.firstname, + "lastname": currentUser.lastname, + "enabled": currentUser.enabled, + "isSysAdmin": getattr(currentUser, "isSysAdmin", False), + "createdAt": getattr(currentUser, "createdAt", None), + "updatedAt": getattr(currentUser, "updatedAt", None), + "lastLogin": getattr(currentUser, "lastLogin", None), + "language": getattr(currentUser, "language", None), + "authenticationAuthority": str(getattr(currentUser, "authenticationAuthority", "")) + } + + # Mandate memberships + from modules.datamodels.datamodelMembership import UserMandate + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"userId": str(currentUser.id)} + ) + + mandates = [] + for um in userMandates: + mandateId = um.get("mandateId") + + # Get mandate details + from modules.datamodels.datamodelUam import Mandate + mandateRecords = rootInterface.db.getRecordset( + Mandate, + recordFilter={"id": mandateId} + ) + mandateName = mandateRecords[0].get("name") if mandateRecords else "Unknown" + + # Get roles for this membership + roleIds = rootInterface.getRoleIdsForUserMandate(um.get("id")) + + mandates.append({ + "userMandateId": um.get("id"), + "mandateId": mandateId, + "mandateName": mandateName, + "enabled": um.get("enabled", True), + "roleIds": roleIds, + "joinedAt": um.get("createdAt") + }) + + # Feature access records + from modules.datamodels.datamodelMembership import FeatureAccess + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": str(currentUser.id)} + ) + + featureAccessList = [] + for fa in featureAccesses: + instanceId = fa.get("featureInstanceId") + + # Get instance details + from modules.datamodels.datamodelFeatures import FeatureInstance + instanceRecords = rootInterface.db.getRecordset( + FeatureInstance, + recordFilter={"id": instanceId} + ) + + instanceInfo = instanceRecords[0] if instanceRecords else {} + roleIds = rootInterface.getRoleIdsForFeatureAccess(fa.get("id")) + + featureAccessList.append({ + "featureAccessId": fa.get("id"), + "featureInstanceId": instanceId, + "featureCode": instanceInfo.get("featureCode"), + "instanceLabel": instanceInfo.get("label"), + "enabled": fa.get("enabled", True), + "roleIds": roleIds + }) + + # Invitations created by user + from modules.datamodels.datamodelInvitation import Invitation + invitationsCreated = rootInterface.db.getRecordset( + Invitation, + recordFilter={"createdBy": str(currentUser.id)} + ) + + invitationsCreatedList = [ + { + "id": inv.get("id"), + "mandateId": inv.get("mandateId"), + "createdAt": inv.get("createdAt"), + "expiresAt": inv.get("expiresAt"), + "maxUses": inv.get("maxUses"), + "currentUses": inv.get("currentUses") + } + for inv in invitationsCreated + ] + + # Invitations used by user + invitationsUsed = rootInterface.db.getRecordset( + Invitation, + recordFilter={"usedBy": str(currentUser.id)} + ) + + invitationsUsedList = [ + { + "id": inv.get("id"), + "mandateId": inv.get("mandateId"), + "usedAt": inv.get("usedAt") + } + for inv in invitationsUsed + ] + + # Audit log + audit_logger.logSecurityEvent( + userId=str(currentUser.id), + mandateId="system", + action="gdpr_data_export", + details="User requested data export (Article 15)" + ) + + logger.info(f"User {currentUser.id} exported personal data (GDPR Art. 15)") + + return DataExportResponse( + exportedAt=getUtcTimestamp(), + userId=str(currentUser.id), + userData=userData, + mandates=mandates, + featureAccesses=featureAccessList, + invitationsCreated=invitationsCreatedList, + invitationsUsed=invitationsUsedList + ) + + except Exception as e: + logger.error(f"Error exporting user data: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to export data: {str(e)}" + ) + + +# ============================================================================= +# Article 20: Right to Data Portability +# ============================================================================= + +@router.get("/data-portability") +@limiter.limit("5/minute") +async def exportPortableData( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> JSONResponse: + """ + Export data in portable, machine-readable format (GDPR Article 20). + + Returns data in JSON-LD format suitable for transfer to another service. + This is a structured format that can be easily parsed by machines. + """ + try: + # Get full export data first + rootInterface = getRootInterface() + + # Build portable data structure + portableData = { + "@context": "https://schema.org", + "@type": "Person", + "identifier": str(currentUser.id), + "name": f"{currentUser.firstname or ''} {currentUser.lastname or ''}".strip() or currentUser.username, + "email": currentUser.email, + "additionalProperty": [] + } + + # Add profile properties + if currentUser.firstname: + portableData["givenName"] = currentUser.firstname + if currentUser.lastname: + portableData["familyName"] = currentUser.lastname + + # Add mandate memberships as organization affiliations + from modules.datamodels.datamodelMembership import UserMandate + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"userId": str(currentUser.id)} + ) + + affiliations = [] + for um in userMandates: + from modules.datamodels.datamodelUam import Mandate + mandateRecords = rootInterface.db.getRecordset( + Mandate, + recordFilter={"id": um.get("mandateId")} + ) + if mandateRecords: + mandate = mandateRecords[0] + affiliations.append({ + "@type": "Organization", + "identifier": um.get("mandateId"), + "name": mandate.get("name"), + "membershipActive": um.get("enabled", True) + }) + + if affiliations: + portableData["affiliation"] = affiliations + + # Wrap in export envelope + exportEnvelope = { + "@context": "https://schema.org", + "@type": "DataDownload", + "identifier": f"export-{currentUser.id}-{int(getUtcTimestamp())}", + "dateCreated": _timestampToIso(getUtcTimestamp()), + "encodingFormat": "application/ld+json", + "about": portableData + } + + # Audit log + audit_logger.logSecurityEvent( + userId=str(currentUser.id), + mandateId="system", + action="gdpr_data_portability", + details="User requested portable data export (Article 20)" + ) + + logger.info(f"User {currentUser.id} exported portable data (GDPR Art. 20)") + + return JSONResponse( + content=exportEnvelope, + media_type="application/ld+json" + ) + + except Exception as e: + logger.error(f"Error exporting portable data: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to export data: {str(e)}" + ) + + +# ============================================================================= +# Article 17: Right to Erasure +# ============================================================================= + +@router.delete("/", response_model=DeletionResult) +@limiter.limit("1/hour") +async def deleteAccount( + request: Request, + confirmDeletion: bool = False, + currentUser: User = Depends(getCurrentUser) +) -> DeletionResult: + """ + Delete own account and all associated data (GDPR Article 17). + + IMPORTANT: This action is irreversible! + - All user data will be permanently deleted + - All mandate memberships will be removed + - All feature accesses will be removed + - All created invitations will be revoked + + Args: + confirmDeletion: Must be True to confirm deletion + """ + if not confirmDeletion: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Deletion not confirmed. Set confirmDeletion=true to proceed." + ) + + # Prevent SysAdmin self-deletion (safety measure) + if getattr(currentUser, "isSysAdmin", False): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="SysAdmin accounts cannot be self-deleted. Contact another SysAdmin." + ) + + try: + rootInterface = getRootInterface() + deletedData = [] + + # 1. Revoke all invitations created by user + from modules.datamodels.datamodelInvitation import Invitation + userInvitations = rootInterface.db.getRecordset( + Invitation, + recordFilter={"createdBy": str(currentUser.id)} + ) + + for inv in userInvitations: + rootInterface.db.recordUpdate( + Invitation, + inv.get("id"), + {"revokedAt": getUtcTimestamp()} + ) + deletedData.append(f"Invitations revoked: {len(userInvitations)}") + + # 2. Delete feature accesses (CASCADE will delete FeatureAccessRoles) + from modules.datamodels.datamodelMembership import FeatureAccess + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": str(currentUser.id)} + ) + + for fa in featureAccesses: + rootInterface.db.recordDelete(FeatureAccess, fa.get("id")) + deletedData.append(f"Feature accesses deleted: {len(featureAccesses)}") + + # 3. Delete mandate memberships (CASCADE will delete UserMandateRoles) + from modules.datamodels.datamodelMembership import UserMandate + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"userId": str(currentUser.id)} + ) + + for um in userMandates: + rootInterface.db.recordDelete(UserMandate, um.get("id")) + deletedData.append(f"Mandate memberships deleted: {len(userMandates)}") + + # 4. Delete active tokens + from modules.datamodels.datamodelSecurity import Token + userTokens = rootInterface.db.getRecordset( + Token, + recordFilter={"userId": str(currentUser.id)} + ) + + for token in userTokens: + rootInterface.db.recordDelete(Token, token.get("id")) + deletedData.append(f"Tokens deleted: {len(userTokens)}") + + # 5. Delete user connections (OAuth) + from modules.datamodels.datamodelUam import UserConnection + userConnections = rootInterface.db.getRecordset( + UserConnection, + recordFilter={"userId": str(currentUser.id)} + ) + + for conn in userConnections: + rootInterface.db.recordDelete(UserConnection, conn.get("id")) + deletedData.append(f"Connections deleted: {len(userConnections)}") + + # 6. Finally, delete the user + deletedAt = getUtcTimestamp() + rootInterface.db.recordDelete(User, str(currentUser.id)) + deletedData.append("User account deleted") + + # Audit log (before user is deleted) + audit_logger.logSecurityEvent( + userId=str(currentUser.id), + mandateId="system", + action="gdpr_account_deletion", + details=f"User deleted own account (Article 17). Data: {', '.join(deletedData)}" + ) + + logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17)") + + return DeletionResult( + success=True, + userId=str(currentUser.id), + deletedAt=deletedAt, + deletedData=deletedData, + message="Account and all associated data have been permanently deleted." + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting account: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to delete account: {str(e)}" + ) + + +# ============================================================================= +# Consent Information Endpoint +# ============================================================================= + +@router.get("/consent-info", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def getConsentInfo( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Get information about data processing and user rights (GDPR transparency). + + Returns information about: + - What data is collected + - How data is processed + - User rights under GDPR + - Contact information for data protection inquiries + """ + return { + "dataCollected": { + "profile": "Name, email, username, language preferences", + "authentication": "Login timestamps, authentication provider", + "memberships": "Mandate and feature access records", + "activity": "Audit logs for security-relevant actions" + }, + "dataProcessing": { + "purpose": "Providing multi-tenant platform services", + "legalBasis": "Contract fulfillment and legitimate interest", + "retention": "Data retained while account is active, deleted upon account deletion" + }, + "userRights": { + "access": "GET /api/user/me/data-export (Article 15)", + "portability": "GET /api/user/me/data-portability (Article 20)", + "erasure": "DELETE /api/user/me (Article 17)", + "rectification": "PUT /api/local/me (Article 16)" + }, + "contact": { + "email": "privacy@example.com", + "note": "For data protection inquiries, please contact us with your user ID" + } + } + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _timestampToIso(timestamp: float) -> str: + """Convert Unix timestamp to ISO 8601 format""" + from datetime import datetime, timezone + dt = datetime.fromtimestamp(timestamp, tz=timezone.utc) + return dt.isoformat() diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py new file mode 100644 index 00000000..f649d2b2 --- /dev/null +++ b/modules/routes/routeInvitations.py @@ -0,0 +1,812 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Invitation routes for the backend API. +Implements token-based user invitations for self-service onboarding. + +Multi-Tenant Design: +- Invitations are mandate-scoped (Mandate Admin creates them) +- Tokens are secure, time-limited, and optionally use-limited +- Users accept invitations to join mandates/features with predefined roles +""" + +from fastapi import APIRouter, HTTPException, Depends, Request, Query +from typing import List, Dict, Any, Optional +from fastapi import status +import logging +from pydantic import BaseModel, Field + +from modules.auth import limiter, getRequestContext, RequestContext, getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelInvitation import Invitation +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/invitations", + tags=["Invitations"], + responses={404: {"description": "Not found"}} +) + + +# ============================================================================= +# Request/Response Models +# ============================================================================= + +class InvitationCreate(BaseModel): + """Request model for creating an invitation""" + email: Optional[str] = Field(None, description="Target email address (optional)") + roleIds: List[str] = Field(..., description="Role IDs to assign to the invited user") + featureInstanceId: Optional[str] = Field(None, description="Optional feature instance access") + expiresInHours: int = Field( + 72, + ge=1, + le=720, # Max 30 days + description="Hours until invitation expires" + ) + maxUses: int = Field( + 1, + ge=1, + le=100, + description="Maximum number of times this invitation can be used" + ) + + +class InvitationResponse(BaseModel): + """Response model for invitation""" + id: str + token: str + mandateId: str + featureInstanceId: Optional[str] + roleIds: List[str] + email: Optional[str] + createdBy: str + createdAt: float + expiresAt: float + usedBy: Optional[str] + usedAt: Optional[float] + revokedAt: Optional[float] + maxUses: int + currentUses: int + inviteUrl: str # Full URL for the invitation + + +class InvitationValidation(BaseModel): + """Response model for invitation validation""" + valid: bool + reason: Optional[str] + mandateId: Optional[str] + featureInstanceId: Optional[str] + roleIds: List[str] + + +class RegisterAndAcceptRequest(BaseModel): + """Request model for combined registration + invitation acceptance""" + token: str = Field(..., description="Invitation token") + username: str = Field(..., min_length=3, max_length=50, description="Username for the new account") + email: str = Field(..., description="Email address") + password: str = Field(..., min_length=8, description="Password (min 8 characters)") + firstname: Optional[str] = Field(None, description="First name") + lastname: Optional[str] = Field(None, description="Last name") + + +class RegisterAndAcceptResponse(BaseModel): + """Response model for combined registration + invitation acceptance""" + message: str + userId: str + mandateId: str + userMandateId: str + featureAccessId: Optional[str] + roleIds: List[str] + + +# ============================================================================= +# Invitation CRUD Endpoints +# ============================================================================= + +@router.post("/", response_model=InvitationResponse) +@limiter.limit("30/minute") +async def createInvitation( + request: Request, + data: InvitationCreate, + context: RequestContext = Depends(getRequestContext) +) -> InvitationResponse: + """ + Create a new invitation for the current mandate. + + Requires Mandate-Admin role. Creates a secure token that can be shared + with users to join the mandate with predefined roles. + + Args: + data: Invitation creation data + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to create invitations" + ) + + try: + rootInterface = getRootInterface() + + # Validate role IDs exist and belong to this mandate or are global + for roleId in data.roleIds: + from modules.datamodels.datamodelRbac import Role + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roleRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Role '{roleId}' not found" + ) + role = roleRecords[0] + # Role must be global or belong to this mandate + roleMandateId = role.get("mandateId") + if roleMandateId and str(roleMandateId) != str(context.mandateId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role '{roleId}' belongs to a different mandate" + ) + + # Validate feature instance if provided + if data.featureInstanceId: + from modules.datamodels.datamodelFeatures import FeatureInstance + instanceRecords = rootInterface.db.getRecordset( + FeatureInstance, + recordFilter={"id": data.featureInstanceId} + ) + if not instanceRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{data.featureInstanceId}' not found" + ) + instance = instanceRecords[0] + if str(instance.get("mandateId")) != str(context.mandateId): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Feature instance belongs to a different mandate" + ) + + # Calculate expiration time + currentTime = getUtcTimestamp() + expiresAt = currentTime + (data.expiresInHours * 3600) + + # Create invitation + invitation = Invitation( + mandateId=str(context.mandateId), + featureInstanceId=data.featureInstanceId, + roleIds=data.roleIds, + email=data.email, + createdBy=str(context.user.id), + expiresAt=expiresAt, + maxUses=data.maxUses + ) + + createdRecord = rootInterface.db.recordCreate(Invitation, invitation.model_dump()) + if not createdRecord: + raise ValueError("Failed to create invitation record") + + # Build invite URL + from modules.shared.configuration import APP_CONFIG + frontendUrl = APP_CONFIG.get("APP_FRONTEND_URL", "http://localhost:8080") + inviteUrl = f"{frontendUrl}/invite/{invitation.token}" + + logger.info( + f"User {context.user.id} created invitation for mandate {context.mandateId}, " + f"expires in {data.expiresInHours}h" + ) + + return InvitationResponse( + id=str(createdRecord.get("id")), + token=str(createdRecord.get("token")), + mandateId=str(createdRecord.get("mandateId")), + featureInstanceId=createdRecord.get("featureInstanceId"), + roleIds=createdRecord.get("roleIds", []), + email=createdRecord.get("email"), + createdBy=str(createdRecord.get("createdBy")), + createdAt=createdRecord.get("createdAt"), + expiresAt=createdRecord.get("expiresAt"), + usedBy=createdRecord.get("usedBy"), + usedAt=createdRecord.get("usedAt"), + revokedAt=createdRecord.get("revokedAt"), + maxUses=createdRecord.get("maxUses", 1), + currentUses=createdRecord.get("currentUses", 0), + inviteUrl=inviteUrl + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating invitation: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create invitation: {str(e)}" + ) + + +@router.get("/", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def listInvitations( + request: Request, + includeUsed: bool = Query(False, description="Include already used invitations"), + includeExpired: bool = Query(False, description="Include expired invitations"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + List invitations for the current mandate. + + Requires Mandate-Admin role. Returns all invitations created for this mandate. + + Args: + includeUsed: Include invitations that have reached maxUses + includeExpired: Include expired invitations + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to list invitations" + ) + + try: + rootInterface = getRootInterface() + + # Get all invitations for this mandate + allInvitations = rootInterface.db.getRecordset( + Invitation, + recordFilter={"mandateId": str(context.mandateId)} + ) + + currentTime = getUtcTimestamp() + result = [] + + for inv in allInvitations: + # Skip revoked invitations + if inv.get("revokedAt"): + continue + + # Filter by usage + if not includeUsed and inv.get("currentUses", 0) >= inv.get("maxUses", 1): + continue + + # Filter by expiration + if not includeExpired and inv.get("expiresAt", 0) < currentTime: + continue + + # Build invite URL + from modules.shared.configuration import APP_CONFIG + frontendUrl = APP_CONFIG.get("APP_FRONTEND_URL", "http://localhost:8080") + inviteUrl = f"{frontendUrl}/invite/{inv.get('token')}" + + result.append({ + **{k: v for k, v in inv.items() if not k.startswith("_")}, + "inviteUrl": inviteUrl, + "isExpired": inv.get("expiresAt", 0) < currentTime, + "isUsedUp": inv.get("currentUses", 0) >= inv.get("maxUses", 1) + }) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing invitations: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list invitations: {str(e)}" + ) + + +@router.delete("/{invitationId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +async def revokeInvitation( + request: Request, + invitationId: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, str]: + """ + Revoke an invitation. + + Requires Mandate-Admin role. Revoked invitations cannot be used. + + Args: + invitationId: Invitation ID + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to revoke invitations" + ) + + try: + rootInterface = getRootInterface() + + # Get invitation + invitationRecords = rootInterface.db.getRecordset( + Invitation, + recordFilter={"id": invitationId} + ) + + if not invitationRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Invitation '{invitationId}' not found" + ) + + invitation = invitationRecords[0] + + # Verify mandate access + if str(invitation.get("mandateId")) != str(context.mandateId): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this invitation" + ) + + # Already revoked? + if invitation.get("revokedAt"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation is already revoked" + ) + + # Revoke invitation + rootInterface.db.recordUpdate( + Invitation, + invitationId, + {"revokedAt": getUtcTimestamp()} + ) + + logger.info(f"User {context.user.id} revoked invitation {invitationId}") + + return {"message": "Invitation revoked", "invitationId": invitationId} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error revoking invitation {invitationId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to revoke invitation: {str(e)}" + ) + + +# ============================================================================= +# Public Invitation Endpoints (No auth required for validation) +# ============================================================================= + +@router.get("/validate/{token}", response_model=InvitationValidation) +@limiter.limit("30/minute") +async def validateInvitation( + request: Request, + token: str +) -> InvitationValidation: + """ + Validate an invitation token (public endpoint). + + Used by the frontend to check if an invitation is valid before + showing the registration/acceptance form. + + Args: + token: Invitation token + """ + try: + rootInterface = getRootInterface() + + # Find invitation by token + invitationRecords = rootInterface.db.getRecordset( + Invitation, + recordFilter={"token": token} + ) + + if not invitationRecords: + return InvitationValidation( + valid=False, + reason="Invitation not found", + mandateId=None, + featureInstanceId=None, + roleIds=[] + ) + + invitation = invitationRecords[0] + + # Check if revoked + if invitation.get("revokedAt"): + return InvitationValidation( + valid=False, + reason="Invitation has been revoked", + mandateId=None, + featureInstanceId=None, + roleIds=[] + ) + + # Check if expired + currentTime = getUtcTimestamp() + if invitation.get("expiresAt", 0) < currentTime: + return InvitationValidation( + valid=False, + reason="Invitation has expired", + mandateId=None, + featureInstanceId=None, + roleIds=[] + ) + + # Check if used up + if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): + return InvitationValidation( + valid=False, + reason="Invitation has reached maximum uses", + mandateId=None, + featureInstanceId=None, + roleIds=[] + ) + + return InvitationValidation( + valid=True, + reason=None, + mandateId=invitation.get("mandateId"), + featureInstanceId=invitation.get("featureInstanceId"), + roleIds=invitation.get("roleIds", []) + ) + + except Exception as e: + logger.error(f"Error validating invitation token: {e}") + return InvitationValidation( + valid=False, + reason="Validation error", + mandateId=None, + featureInstanceId=None, + roleIds=[] + ) + + +@router.post("/accept/{token}", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def acceptInvitation( + request: Request, + token: str, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Accept an invitation (requires authentication). + + The authenticated user joins the mandate with the predefined roles. + If the user is already a member, their roles are updated. + + Args: + token: Invitation token + """ + try: + rootInterface = getRootInterface() + + # Find invitation by token + invitationRecords = rootInterface.db.getRecordset( + Invitation, + recordFilter={"token": token} + ) + + if not invitationRecords: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Invitation not found" + ) + + invitation = invitationRecords[0] + + # Validate invitation + if invitation.get("revokedAt"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has been revoked" + ) + + currentTime = getUtcTimestamp() + if invitation.get("expiresAt", 0) < currentTime: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has expired" + ) + + if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has reached maximum uses" + ) + + mandateId = invitation.get("mandateId") + roleIds = invitation.get("roleIds", []) + featureInstanceId = invitation.get("featureInstanceId") + + # Check if user is already a member + existingMembership = rootInterface.getUserMandate(str(currentUser.id), mandateId) + + if existingMembership: + # Update existing membership with additional roles + for roleId in roleIds: + try: + rootInterface.addRoleToUserMandate(str(existingMembership.id), roleId) + except Exception: + pass # Role might already be assigned + + userMandateId = str(existingMembership.id) + message = "Roles updated for existing membership" + else: + # Create new membership + userMandate = rootInterface.createUserMandate( + userId=str(currentUser.id), + mandateId=mandateId, + roleIds=roleIds + ) + userMandateId = str(userMandate.id) + message = "Successfully joined mandate" + + # Grant feature access if specified + featureAccessId = None + if featureInstanceId: + existingAccess = rootInterface.getFeatureAccess(str(currentUser.id), featureInstanceId) + if not existingAccess: + # Create feature access with instance-level roles if any + instanceRoleIds = [r for r in roleIds if _isInstanceRole(rootInterface, r, featureInstanceId)] + featureAccess = rootInterface.createFeatureAccess( + userId=str(currentUser.id), + featureInstanceId=featureInstanceId, + roleIds=instanceRoleIds + ) + featureAccessId = str(featureAccess.id) + + # Update invitation usage + rootInterface.db.recordUpdate( + Invitation, + invitation.get("id"), + { + "currentUses": invitation.get("currentUses", 0) + 1, + "usedBy": str(currentUser.id), + "usedAt": currentTime + } + ) + + logger.info( + f"User {currentUser.id} accepted invitation {invitation.get('id')} " + f"for mandate {mandateId}" + ) + + return { + "message": message, + "mandateId": mandateId, + "userMandateId": userMandateId, + "featureAccessId": featureAccessId, + "roleIds": roleIds + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error accepting invitation: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to accept invitation: {str(e)}" + ) + + +# ============================================================================= +# Combined Registration + Accept Invitation +# ============================================================================= + +@router.post("/register-and-accept", response_model=RegisterAndAcceptResponse) +@limiter.limit("10/minute") # Stricter rate limit for registration +async def registerAndAcceptInvitation( + request: Request, + data: RegisterAndAcceptRequest +) -> RegisterAndAcceptResponse: + """ + Combined endpoint: Register a new user AND accept an invitation in one step. + + This is a PUBLIC endpoint - no authentication required. + + Flow: + 1. Validate invitation token + 2. Check email matches (if invitation has email restriction) + 3. Create new user account + 4. Create UserMandate membership with roles + 5. Optionally grant FeatureAccess + 6. Update invitation usage + + The user can then login with their new credentials. + """ + try: + rootInterface = getRootInterface() + + # 1. Validate invitation + invitation = rootInterface.getInvitationByToken(data.token) + if not invitation: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Invalid invitation token" + ) + + if invitation.get("revokedAt"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has been revoked" + ) + + currentTime = getUtcTimestamp() + if invitation.get("expiresAt", 0) < currentTime: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has expired" + ) + + if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has reached maximum uses" + ) + + # 2. Check email restriction + invitationEmail = invitation.get("email") + if invitationEmail and invitationEmail.lower() != data.email.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email does not match the invitation" + ) + + # 3. Check if username or email already exists + existingUsername = rootInterface.getUserByUsername(data.username) + if existingUsername: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already exists" + ) + + existingEmail = rootInterface.getUserByEmail(data.email) + if existingEmail: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Email already registered. Please login and accept the invitation." + ) + + # 4. Create new user + from modules.security.passwordUtils import hashPassword + hashedPassword = hashPassword(data.password) + + newUser = rootInterface.createUser( + username=data.username, + email=data.email, + passwordHash=hashedPassword, + firstname=data.firstname, + lastname=data.lastname + ) + + if not newUser: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to create user account" + ) + + userId = str(newUser.id) + mandateId = invitation.get("mandateId") + roleIds = invitation.get("roleIds", []) + featureInstanceId = invitation.get("featureInstanceId") + + # 5. Create UserMandate membership + userMandate = rootInterface.createUserMandate( + userId=userId, + mandateId=mandateId, + roleIds=roleIds + ) + userMandateId = str(userMandate.id) + + # 6. Grant feature access if specified + featureAccessId = None + if featureInstanceId: + instanceRoleIds = [r for r in roleIds if _isInstanceRole(rootInterface, r, featureInstanceId)] + featureAccess = rootInterface.createFeatureAccess( + userId=userId, + featureInstanceId=featureInstanceId, + roleIds=instanceRoleIds + ) + featureAccessId = str(featureAccess.id) + + # 7. Update invitation usage + rootInterface.db.recordUpdate( + Invitation, + invitation.get("id"), + { + "currentUses": invitation.get("currentUses", 0) + 1, + "usedBy": userId, + "usedAt": currentTime + } + ) + + logger.info( + f"New user {userId} registered and accepted invitation {invitation.get('id')} " + f"for mandate {mandateId}" + ) + + return RegisterAndAcceptResponse( + message="Account created and invitation accepted successfully", + userId=userId, + mandateId=mandateId, + userMandateId=userMandateId, + featureAccessId=featureAccessId, + roleIds=roleIds + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in register-and-accept: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to complete registration: {str(e)}" + ) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _hasMandateAdminRole(context: RequestContext) -> bool: + """ + Check if the user has mandate admin role in the current context. + """ + if context.isSysAdmin: + return True + + if not context.roleIds: + return False + + try: + rootInterface = getRootInterface() + from modules.datamodels.datamodelRbac import Role + + for roleId in context.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + # Admin role at mandate level + if roleLabel == "admin" and role.get("mandateId") and not role.get("featureInstanceId"): + return True + + return False + + except Exception as e: + logger.error(f"Error checking mandate admin role: {e}") + return False + + +def _isInstanceRole(interface, roleId: str, featureInstanceId: str) -> bool: + """ + Check if a role belongs to a specific feature instance. + """ + try: + from modules.datamodels.datamodelRbac import Role + roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + return str(role.get("featureInstanceId", "")) == str(featureInstanceId) + return False + except Exception: + return False diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index 9222a4bd..dae6e04e 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -7,7 +7,8 @@ import logging import json # Import auth module -from modules.auth import limiter, getCurrentUser +from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext +from modules.datamodels.datamodelRbac import Role # Import interfaces import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects @@ -379,16 +380,23 @@ async def triggerSubscription( request: Request, subscriptionId: str = Path(..., description="ID of the subscription to trigger"), eventParameters: Dict[str, Any] = Body(...), - currentUser: User = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> MessagingSubscriptionExecutionResult: - """Trigger a subscription with event parameters""" - # RBAC-Check: Nur Admin/Mandate-Admin kann triggern - # TODO: Add proper RBAC check here + """ + Trigger a subscription with event parameters. + + Requires Mandate-Admin role or SysAdmin. + """ + # RBAC-Check: Admin or Mandate-Admin can trigger + if not _hasTriggerPermission(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin or Mandate-Admin role required to trigger subscriptions" + ) # Get messaging service from request app state - # We need to access services through the request from modules.services import getInterface as getServicesInterface - services = getServicesInterface(currentUser, None) + services = getServicesInterface(context.user, None, mandateId=str(context.mandateId)) # Konvertiere Dict zu Pydantic Model eventParams = MessagingEventParameters(triggerData=eventParameters) @@ -397,6 +405,37 @@ async def triggerSubscription( return executionResult +def _hasTriggerPermission(context: RequestContext) -> bool: + """ + Check if user has permission to trigger subscriptions. + Requires admin or mandate-admin role. + """ + if context.isSysAdmin: + return True + + if not context.roleIds: + return False + + try: + from modules.interfaces.interfaceDbAppObjects import getRootInterface + rootInterface = getRootInterface() + + for roleId in context.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + # Admin role at mandate level or system admin + if roleLabel in ("admin", "sysadmin"): + return True + + return False + + except Exception as e: + logger.error(f"Error checking trigger permission: {e}") + return False + + # Delivery Endpoints @router.get("/deliveries", response_model=PaginatedResponse[MessagingDelivery]) diff --git a/modules/routes/routeRbac.py b/modules/routes/routeRbac.py index 3c940b72..d2e92c82 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeRbac.py @@ -3,6 +3,11 @@ """ RBAC routes for the backend API. Implements endpoints for role-based access control permissions. + +MULTI-TENANT: +- Permission queries use RequestContext (mandateId from header) +- AccessRule management is SysAdmin-only (system resources) +- Role management is SysAdmin-only (system resources) """ from fastapi import APIRouter, HTTPException, Depends, Query, Body, Path, Request @@ -11,11 +16,11 @@ import logging import json import math -from modules.auth import getCurrentUser, limiter +from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) @@ -33,10 +38,11 @@ async def getPermissions( request: Request, context: str = Query(..., description="Context type: DATA, UI, or RESOURCE"), item: Optional[str] = Query(None, description="Item identifier (table name, UI path, or resource path)"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(getRequestContext) ) -> UserPermissions: """ Get RBAC permissions for the current user for a specific context and item. + MULTI-TENANT: Uses RequestContext (mandateId/featureInstanceId from headers). Query Parameters: - context: Context type (DATA, UI, or RESOURCE) @@ -63,16 +69,17 @@ async def getPermissions( ) # Get interface and RBAC permissions - interface = getInterface(currentUser) + interface = getInterface(reqContext.user) if not interface.rbac: raise HTTPException( status_code=500, detail="RBAC interface not available" ) - # Get permissions + # MULTI-TENANT: Get permissions using context (mandateId/featureInstanceId) + # For now, pass user - RBAC will be extended to use context in later phases permissions = interface.rbac.getUserPermissions( - currentUser, + reqContext.user, accessContext, item or "" ) @@ -94,10 +101,11 @@ async def getPermissions( async def getAllPermissions( request: Request, context: Optional[str] = Query(None, description="Context type: UI or RESOURCE (if not provided, returns both)"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ Get all RBAC permissions for the current user for UI and/or RESOURCE contexts. + MULTI-TENANT: Uses RequestContext (mandateId/featureInstanceId from headers). This endpoint is optimized for UI initialization to avoid multiple API calls. Query Parameters: @@ -128,7 +136,7 @@ async def getAllPermissions( """ try: # Get interface and RBAC permissions - interface = getInterface(currentUser) + interface = getInterface(reqContext.user) if not interface.rbac: raise HTTPException( status_code=500, @@ -158,9 +166,9 @@ async def getAllPermissions( result: Dict[str, Any] = {} - # Get all access rules for user's roles - roleLabels = currentUser.roleLabels or [] - if not roleLabels: + # MULTI-TENANT: Get role IDs from context (computed from mandateId/featureInstanceId) + roleIds = reqContext.roleIds or [] + if not roleIds and not reqContext.isSysAdmin: # User has no roles, return empty permissions for ctx in contextsToFetch: result[ctx.value.lower()] = {} @@ -171,9 +179,9 @@ async def getAllPermissions( for ctx in contextsToFetch: allRules[ctx] = [] # Get all rules for user's roles in this context - for roleLabel in roleLabels: + for roleId in roleIds: rules = interface.getAccessRules( - roleLabel=roleLabel, + roleId=str(roleId), context=ctx, pagination=None ) @@ -191,7 +199,7 @@ async def getAllPermissions( # For each item, calculate user permissions for item in sorted(items): - permissions = interface.rbac.getUserPermissions(currentUser, ctx, item) + permissions = interface.rbac.getUserPermissions(reqContext.user, ctx, item) # Only include if user has view permission if permissions.view: result[ctx.value.lower()][item] = { @@ -222,11 +230,11 @@ async def getAccessRules( context: Optional[str] = Query(None, description="Filter by context (DATA, UI, RESOURCE)"), item: Optional[str] = Query(None, description="Filter by item identifier"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> PaginatedResponse: """ Get access rules with optional filters. - Only returns rules that the current user has permission to view. + MULTI-TENANT: SysAdmin-only (AccessRules are system resources). Query Parameters: - roleLabel: Optional role label filter @@ -237,29 +245,8 @@ async def getAccessRules( - List of AccessRule objects """ try: - # Get interface - interface = getInterface(currentUser) - - # Check if user has permission to view access rules - # For now, only sysadmin can view rules - if not interface.rbac: - raise HTTPException( - status_code=500, - detail="RBAC interface not available" - ) - - # Check permission - only sysadmin can view rules - permissions = interface.rbac.getUserPermissions( - currentUser, - AccessRuleContext.DATA, - "AccessRule" - ) - - if not permissions.view or permissions.read == AccessLevel.NONE: - raise HTTPException( - status_code=403, - detail="No permission to view access rules" - ) + # Get interface - SysAdmin uses root interface + interface = getRootInterface() # Parse context if provided accessContext = None @@ -329,11 +316,11 @@ async def getAccessRules( async def getAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> dict: """ Get a specific access rule by ID. - Only returns rule if the current user has permission to view it. + MULTI-TENANT: SysAdmin-only. Path Parameters: - ruleId: Access rule ID @@ -342,28 +329,8 @@ async def getAccessRule( - AccessRule object """ try: - # Get interface - interface = getInterface(currentUser) - - # Check if user has permission to view access rules - if not interface.rbac: - raise HTTPException( - status_code=500, - detail="RBAC interface not available" - ) - - # Check permission - only sysadmin can view rules - permissions = interface.rbac.getUserPermissions( - currentUser, - AccessRuleContext.DATA, - "AccessRule" - ) - - if not permissions.view or permissions.read == AccessLevel.NONE: - raise HTTPException( - status_code=403, - detail="No permission to view access rules" - ) + # Get interface - SysAdmin uses root interface + interface = getRootInterface() # Get rule rule = interface.getAccessRule(ruleId) @@ -391,11 +358,11 @@ async def getAccessRule( async def createAccessRule( request: Request, accessRuleData: dict = Body(..., description="Access rule data"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> dict: """ Create a new access rule. - Only sysadmin can create access rules. + MULTI-TENANT: SysAdmin-only. Request Body: - AccessRule object data (roleLabel, context, item, view, read, create, update, delete) @@ -404,28 +371,8 @@ async def createAccessRule( - Created AccessRule object """ try: - # Get interface - interface = getInterface(currentUser) - - # Check if user has permission to create access rules - if not interface.rbac: - raise HTTPException( - status_code=500, - detail="RBAC interface not available" - ) - - # Check permission - only sysadmin can create rules - permissions = interface.rbac.getUserPermissions( - currentUser, - AccessRuleContext.DATA, - "AccessRule" - ) - - if not permissions.create or permissions.create == AccessLevel.NONE: - raise HTTPException( - status_code=403, - detail="No permission to create access rules" - ) + # Get interface - SysAdmin uses root interface + interface = getRootInterface() # Validate and parse access rule data try: @@ -457,7 +404,7 @@ async def createAccessRule( # Create rule createdRule = interface.createAccessRule(accessRule) - logger.info(f"Created access rule {createdRule.id} by user {currentUser.id}") + logger.info(f"Created access rule {createdRule.id} by SysAdmin {reqContext.user.id}") # Convert to dict for JSON serialization return createdRule.model_dump() @@ -478,11 +425,11 @@ async def updateAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), accessRuleData: dict = Body(..., description="Updated access rule data"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> dict: """ Update an existing access rule. - Only sysadmin can update access rules. + MULTI-TENANT: SysAdmin-only. Path Parameters: - ruleId: Access rule ID @@ -494,28 +441,8 @@ async def updateAccessRule( - Updated AccessRule object """ try: - # Get interface - interface = getInterface(currentUser) - - # Check if user has permission to update access rules - if not interface.rbac: - raise HTTPException( - status_code=500, - detail="RBAC interface not available" - ) - - # Check permission - only sysadmin can update rules - permissions = interface.rbac.getUserPermissions( - currentUser, - AccessRuleContext.DATA, - "AccessRule" - ) - - if not permissions.update or permissions.update == AccessLevel.NONE: - raise HTTPException( - status_code=403, - detail="No permission to update access rules" - ) + # Get interface - SysAdmin uses root interface + interface = getRootInterface() # Get existing rule to ensure it exists existingRule = interface.getAccessRule(ruleId) @@ -560,7 +487,7 @@ async def updateAccessRule( # Update rule updatedRule = interface.updateAccessRule(ruleId, accessRule) - logger.info(f"Updated access rule {ruleId} by user {currentUser.id}") + logger.info(f"Updated access rule {ruleId} by SysAdmin {reqContext.user.id}") # Convert to dict for JSON serialization return updatedRule.model_dump() @@ -580,11 +507,11 @@ async def updateAccessRule( async def deleteAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> dict: """ Delete an access rule. - Only sysadmin can delete access rules. + MULTI-TENANT: SysAdmin-only. Path Parameters: - ruleId: Access rule ID @@ -593,28 +520,8 @@ async def deleteAccessRule( - Success message """ try: - # Get interface - interface = getInterface(currentUser) - - # Check if user has permission to delete access rules - if not interface.rbac: - raise HTTPException( - status_code=500, - detail="RBAC interface not available" - ) - - # Check permission - only sysadmin can delete rules - permissions = interface.rbac.getUserPermissions( - currentUser, - AccessRuleContext.DATA, - "AccessRule" - ) - - if not permissions.delete or permissions.delete == AccessLevel.NONE: - raise HTTPException( - status_code=403, - detail="No permission to delete access rules" - ) + # Get interface - SysAdmin uses root interface + interface = getRootInterface() # Get existing rule to ensure it exists existingRule = interface.getAccessRule(ruleId) @@ -633,7 +540,7 @@ async def deleteAccessRule( detail=f"Failed to delete access rule {ruleId}" ) - logger.info(f"Deleted access rule {ruleId} by user {currentUser.id}") + logger.info(f"Deleted access rule {ruleId} by SysAdmin {reqContext.user.id}") return {"success": True, "message": f"Access rule {ruleId} deleted successfully"} @@ -649,38 +556,26 @@ async def deleteAccessRule( # ============================================================================ # Role Management Endpoints +# MULTI-TENANT: All role management is SysAdmin-only (roles are system resources) # ============================================================================ -def _ensureAdminAccess(currentUser: User) -> None: - """Ensure current user has admin access to RBAC roles management.""" - interface = getInterface(currentUser) - - # Check if user has admin or sysadmin role - roleLabels = currentUser.roleLabels or [] - if "sysadmin" not in roleLabels and "admin" not in roleLabels: - raise HTTPException( - status_code=403, - detail="Admin or sysadmin role required to manage RBAC roles" - ) - @router.get("/roles", response_model=PaginatedResponse) @limiter.limit("60/minute") async def listRoles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> PaginatedResponse: """ Get list of all available roles with metadata. + MULTI-TENANT: SysAdmin-only (roles are system resources). Returns: - List of role dictionaries with role label, description, and user count """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Parse pagination parameter paginationParams = None @@ -696,21 +591,11 @@ async def listRoles( detail=f"Invalid pagination parameter: {str(e)}" ) - # Get all roles from database (without pagination) to enrich with user counts and add custom roles - # Note: We get all roles first because we need to add custom roles before pagination + # Get all roles from database dbRoles = interface.getAllRoles(pagination=None) - # Get all users to count role assignments - # Since _ensureAdminAccess ensures user is sysadmin or admin, - # and getUsersByMandate returns all users for sysadmin regardless of mandateId, - # we can pass the current user's mandateId (for sysadmin it will be ignored by RBAC) - allUsers = interface.getUsersByMandate(currentUser.mandateId or "", pagination=None) - - # Count users per role - roleCounts: Dict[str, int] = {} - for user in allUsers: - for roleLabel in (user.roleLabels or []): - roleCounts[roleLabel] = roleCounts.get(roleLabel, 0) + 1 + # Count role assignments from UserMandateRole table + roleCounts = interface.countRoleAssignments() # Convert Role objects to dictionaries and add user counts result = [] @@ -719,22 +604,10 @@ async def listRoles( "id": role.id, "roleLabel": role.roleLabel, "description": role.description, - "userCount": roleCounts.get(role.roleLabel, 0), + "userCount": roleCounts.get(str(role.id), 0), "isSystemRole": role.isSystemRole }) - # Add any roles found in user assignments that don't exist in database - dbRoleLabels = {role.roleLabel for role in dbRoles} - for roleLabel, count in roleCounts.items(): - if roleLabel not in dbRoleLabels: - result.append({ - "id": None, - "roleLabel": roleLabel, - "description": {"en": f"Custom role: {roleLabel}", "fr": f"Rôle personnalisé : {roleLabel}"}, - "userCount": count, - "isSystemRole": False - }) - # Apply filtering and sorting if pagination requested if paginationParams: # Apply filtering (if filters provided) @@ -789,19 +662,17 @@ async def listRoles( @limiter.limit("60/minute") async def getRoleOptions( request: Request, - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get role options for select dropdowns. - Returns roles in format suitable for frontend select components. + MULTI-TENANT: SysAdmin-only. Returns: - List of role option dictionaries with value and label """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() # Get all roles from database dbRoles = interface.getAllRoles() @@ -833,10 +704,11 @@ async def getRoleOptions( async def createRole( request: Request, role: Role = Body(...), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Create a new role. + MULTI-TENANT: SysAdmin-only. Request Body: - role: Role object to create @@ -845,12 +717,12 @@ async def createRole( - Created role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() createdRole = interface.createRole(role) + logger.info(f"Created role {createdRole.roleLabel} by SysAdmin {reqContext.user.id}") + return { "id": createdRole.id, "roleLabel": createdRole.roleLabel, @@ -878,10 +750,11 @@ async def createRole( async def getRole( request: Request, roleId: str = Path(..., description="Role ID"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get a role by ID. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -890,9 +763,7 @@ async def getRole( - Role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() role = interface.getRole(roleId) if not role: @@ -924,10 +795,11 @@ async def updateRole( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update an existing role. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -939,12 +811,12 @@ async def updateRole( - Updated role dictionary """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() updatedRole = interface.updateRole(roleId, role) + logger.info(f"Updated role {roleId} by SysAdmin {reqContext.user.id}") + return { "id": updatedRole.id, "roleLabel": updatedRole.roleLabel, @@ -972,10 +844,11 @@ async def updateRole( async def deleteRole( request: Request, roleId: str = Path(..., description="Role ID"), - currentUser: User = Depends(getCurrentUser) + reqContext: RequestContext = Depends(requireSysAdmin) ) -> Dict[str, str]: """ Delete a role. + MULTI-TENANT: SysAdmin-only. Path Parameters: - roleId: Role ID @@ -984,9 +857,7 @@ async def deleteRole( - Success message """ try: - _ensureAdminAccess(currentUser) - - interface = getInterface(currentUser) + interface = getRootInterface() success = interface.deleteRole(roleId) if not success: @@ -995,6 +866,8 @@ async def deleteRole( detail=f"Role {roleId} not found" ) + logger.info(f"Deleted role {roleId} by SysAdmin {reqContext.user.id}") + return {"message": f"Role {roleId} deleted successfully"} except HTTPException: diff --git a/modules/routes/routeRbacExport.py b/modules/routes/routeRbacExport.py new file mode 100644 index 00000000..e7cc7204 --- /dev/null +++ b/modules/routes/routeRbacExport.py @@ -0,0 +1,608 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +RBAC export/import routes for the backend API. +Implements endpoints for exporting and importing RBAC configurations. + +Multi-Tenant Design: +- Global templates: SysAdmin can export/import +- Mandate-scoped RBAC: Mandate Admin can export/import +- Feature instance roles: Included in mandate export +""" + +from fastapi import APIRouter, HTTPException, Depends, Request, UploadFile, File +from fastapi.responses import JSONResponse +from typing import List, Dict, Any, Optional +from fastapi import status +import logging +import json +from pydantic import BaseModel, Field + +from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelRbac import Role, AccessRule +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/rbac", + tags=["RBAC Export/Import"], + responses={404: {"description": "Not found"}} +) + + +# ============================================================================= +# Request/Response Models +# ============================================================================= + +class RoleExport(BaseModel): + """Export model for a role with its access rules""" + roleLabel: str + description: Dict[str, str] + featureCode: Optional[str] + isSystemRole: bool + accessRules: List[Dict[str, Any]] + + +class RbacExportData(BaseModel): + """Complete RBAC export data""" + exportVersion: str = "1.0" + exportedAt: float + exportedBy: str + scope: str # "global" or "mandate" + mandateId: Optional[str] + roles: List[RoleExport] + + +class RbacImportResult(BaseModel): + """Result of RBAC import operation""" + rolesCreated: int + rolesUpdated: int + rolesSkipped: int + rulesCreated: int + rulesUpdated: int + errors: List[str] + + +# ============================================================================= +# Global RBAC Export/Import (SysAdmin only) +# ============================================================================= + +@router.get("/export/global", response_model=RbacExportData) +@limiter.limit("10/minute") +async def exportGlobalRbac( + request: Request, + sysAdmin: User = Depends(requireSysAdmin) +) -> RbacExportData: + """ + Export global (template) RBAC rules. + + SysAdmin only - exports template roles that are copied to new feature instances. + These are roles with mandateId=NULL. + """ + try: + rootInterface = getRootInterface() + + # Get all global template roles (mandateId is NULL) + allRoles = rootInterface.db.getRecordset(Role) + globalRoles = [r for r in allRoles if r.get("mandateId") is None] + + exportRoles = [] + for role in globalRoles: + roleId = role.get("id") + + # Get access rules for this role + accessRules = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": roleId} + ) + + exportRoles.append(RoleExport( + roleLabel=role.get("roleLabel"), + description=role.get("description", {}), + featureCode=role.get("featureCode"), + isSystemRole=role.get("isSystemRole", False), + accessRules=[ + { + "context": r.get("context"), + "item": r.get("item"), + "view": r.get("view", False), + "read": r.get("read"), + "create": r.get("create"), + "update": r.get("update"), + "delete": r.get("delete") + } + for r in accessRules + ] + )) + + logger.info(f"SysAdmin {sysAdmin.id} exported global RBAC ({len(exportRoles)} roles)") + + return RbacExportData( + exportedAt=getUtcTimestamp(), + exportedBy=str(sysAdmin.id), + scope="global", + mandateId=None, + roles=exportRoles + ) + + except Exception as e: + logger.error(f"Error exporting global RBAC: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to export RBAC: {str(e)}" + ) + + +@router.post("/import/global", response_model=RbacImportResult) +@limiter.limit("5/minute") +async def importGlobalRbac( + request: Request, + file: UploadFile = File(..., description="JSON file with RBAC export data"), + updateExisting: bool = False, + sysAdmin: User = Depends(requireSysAdmin) +) -> RbacImportResult: + """ + Import global (template) RBAC rules. + + SysAdmin only - imports template roles and their access rules. + + Args: + file: JSON file containing RbacExportData + updateExisting: If True, update existing roles. If False, skip them. + """ + try: + # Read and parse file + content = await file.read() + try: + data = json.loads(content.decode("utf-8")) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid JSON: {str(e)}" + ) + + # Validate structure + if "roles" not in data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing 'roles' field in import data" + ) + + rootInterface = getRootInterface() + result = RbacImportResult( + rolesCreated=0, + rolesUpdated=0, + rolesSkipped=0, + rulesCreated=0, + rulesUpdated=0, + errors=[] + ) + + for roleData in data.get("roles", []): + try: + roleLabel = roleData.get("roleLabel") + featureCode = roleData.get("featureCode") + + if not roleLabel: + result.errors.append(f"Role without label skipped") + result.rolesSkipped += 1 + continue + + # Check if role exists (global role with same label and featureCode) + existingRoles = rootInterface.db.getRecordset( + Role, + recordFilter={ + "roleLabel": roleLabel, + "mandateId": None, + "featureCode": featureCode + } + ) + + if existingRoles: + if updateExisting: + # Update existing role + existingRole = existingRoles[0] + roleId = existingRole.get("id") + + rootInterface.db.recordUpdate( + Role, + roleId, + { + "description": roleData.get("description", {}), + "isSystemRole": roleData.get("isSystemRole", False) + } + ) + + # Update access rules + result.rulesUpdated += _updateAccessRules( + rootInterface, + roleId, + roleData.get("accessRules", []) + ) + + result.rolesUpdated += 1 + else: + result.rolesSkipped += 1 + continue + else: + # Create new role + newRole = Role( + roleLabel=roleLabel, + description=roleData.get("description", {}), + featureCode=featureCode, + mandateId=None, + featureInstanceId=None, + isSystemRole=roleData.get("isSystemRole", False) + ) + + createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) + roleId = createdRole.get("id") + + # Create access rules + for ruleData in roleData.get("accessRules", []): + newRule = AccessRule( + roleId=roleId, + context=ruleData.get("context"), + item=ruleData.get("item"), + view=ruleData.get("view", False), + read=ruleData.get("read"), + create=ruleData.get("create"), + update=ruleData.get("update"), + delete=ruleData.get("delete") + ) + rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) + result.rulesCreated += 1 + + result.rolesCreated += 1 + + except Exception as e: + result.errors.append(f"Error processing role '{roleData.get('roleLabel', 'unknown')}': {str(e)}") + + logger.info( + f"SysAdmin {sysAdmin.id} imported global RBAC: " + f"{result.rolesCreated} created, {result.rolesUpdated} updated, " + f"{result.rolesSkipped} skipped" + ) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error importing global RBAC: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to import RBAC: {str(e)}" + ) + + +# ============================================================================= +# Mandate RBAC Export/Import (Mandate Admin) +# ============================================================================= + +@router.get("/export/mandate", response_model=RbacExportData) +@limiter.limit("10/minute") +async def exportMandateRbac( + request: Request, + includeFeatureInstances: bool = True, + context: RequestContext = Depends(getRequestContext) +) -> RbacExportData: + """ + Export RBAC rules for the current mandate. + + Requires Mandate-Admin role. Exports mandate-level roles and optionally + feature instance roles. + + Args: + includeFeatureInstances: Include feature instance roles in export + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to export RBAC" + ) + + try: + rootInterface = getRootInterface() + + # Get mandate-level roles + allRoles = rootInterface.db.getRecordset(Role) + mandateRoles = [ + r for r in allRoles + if str(r.get("mandateId")) == str(context.mandateId) + ] + + # Filter by feature instance if not including them + if not includeFeatureInstances: + mandateRoles = [r for r in mandateRoles if not r.get("featureInstanceId")] + + exportRoles = [] + for role in mandateRoles: + roleId = role.get("id") + + # Get access rules for this role + accessRules = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": roleId} + ) + + exportRoles.append(RoleExport( + roleLabel=role.get("roleLabel"), + description=role.get("description", {}), + featureCode=role.get("featureCode"), + isSystemRole=role.get("isSystemRole", False), + accessRules=[ + { + "context": r.get("context"), + "item": r.get("item"), + "view": r.get("view", False), + "read": r.get("read"), + "create": r.get("create"), + "update": r.get("update"), + "delete": r.get("delete") + } + for r in accessRules + ] + )) + + logger.info( + f"User {context.user.id} exported mandate {context.mandateId} RBAC " + f"({len(exportRoles)} roles)" + ) + + return RbacExportData( + exportedAt=getUtcTimestamp(), + exportedBy=str(context.user.id), + scope="mandate", + mandateId=str(context.mandateId), + roles=exportRoles + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error exporting mandate RBAC: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to export RBAC: {str(e)}" + ) + + +@router.post("/import/mandate", response_model=RbacImportResult) +@limiter.limit("5/minute") +async def importMandateRbac( + request: Request, + file: UploadFile = File(..., description="JSON file with RBAC export data"), + updateExisting: bool = False, + context: RequestContext = Depends(getRequestContext) +) -> RbacImportResult: + """ + Import RBAC rules for the current mandate. + + Requires Mandate-Admin role. Imports roles as mandate-level roles + (not feature instance roles - those are created via template copying). + + Args: + file: JSON file containing RbacExportData + updateExisting: If True, update existing roles. If False, skip them. + """ + if not context.mandateId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="X-Mandate-Id header is required" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to import RBAC" + ) + + try: + # Read and parse file + content = await file.read() + try: + data = json.loads(content.decode("utf-8")) + except json.JSONDecodeError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid JSON: {str(e)}" + ) + + # Validate structure + if "roles" not in data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing 'roles' field in import data" + ) + + rootInterface = getRootInterface() + result = RbacImportResult( + rolesCreated=0, + rolesUpdated=0, + rolesSkipped=0, + rulesCreated=0, + rulesUpdated=0, + errors=[] + ) + + for roleData in data.get("roles", []): + try: + roleLabel = roleData.get("roleLabel") + featureCode = roleData.get("featureCode") + + if not roleLabel: + result.errors.append(f"Role without label skipped") + result.rolesSkipped += 1 + continue + + # System roles cannot be imported at mandate level + if roleData.get("isSystemRole", False): + result.errors.append(f"System role '{roleLabel}' skipped (SysAdmin only)") + result.rolesSkipped += 1 + continue + + # Check if role exists (mandate role with same label) + existingRoles = rootInterface.db.getRecordset( + Role, + recordFilter={ + "roleLabel": roleLabel, + "mandateId": str(context.mandateId), + "featureInstanceId": None # Only mandate-level roles + } + ) + + if existingRoles: + if updateExisting: + # Update existing role + existingRole = existingRoles[0] + roleId = existingRole.get("id") + + rootInterface.db.recordUpdate( + Role, + roleId, + {"description": roleData.get("description", {})} + ) + + # Update access rules + result.rulesUpdated += _updateAccessRules( + rootInterface, + roleId, + roleData.get("accessRules", []) + ) + + result.rolesUpdated += 1 + else: + result.rolesSkipped += 1 + continue + else: + # Create new role at mandate level + newRole = Role( + roleLabel=roleLabel, + description=roleData.get("description", {}), + featureCode=featureCode, + mandateId=str(context.mandateId), + featureInstanceId=None, + isSystemRole=False # Never create system roles via import + ) + + createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) + roleId = createdRole.get("id") + + # Create access rules + for ruleData in roleData.get("accessRules", []): + newRule = AccessRule( + roleId=roleId, + context=ruleData.get("context"), + item=ruleData.get("item"), + view=ruleData.get("view", False), + read=ruleData.get("read"), + create=ruleData.get("create"), + update=ruleData.get("update"), + delete=ruleData.get("delete") + ) + rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) + result.rulesCreated += 1 + + result.rolesCreated += 1 + + except Exception as e: + result.errors.append(f"Error processing role '{roleData.get('roleLabel', 'unknown')}': {str(e)}") + + logger.info( + f"User {context.user.id} imported mandate {context.mandateId} RBAC: " + f"{result.rolesCreated} created, {result.rolesUpdated} updated, " + f"{result.rolesSkipped} skipped" + ) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error importing mandate RBAC: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to import RBAC: {str(e)}" + ) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _hasMandateAdminRole(context: RequestContext) -> bool: + """ + Check if the user has mandate admin role in the current context. + """ + if context.isSysAdmin: + return True + + if not context.roleIds: + return False + + try: + rootInterface = getRootInterface() + + for roleId in context.roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + roleLabel = role.get("roleLabel", "") + # Admin role at mandate level + if roleLabel == "admin" and role.get("mandateId") and not role.get("featureInstanceId"): + return True + + return False + + except Exception as e: + logger.error(f"Error checking mandate admin role: {e}") + return False + + +def _updateAccessRules(interface, roleId: str, newRules: List[Dict[str, Any]]) -> int: + """ + Update access rules for a role. + Replaces existing rules with new ones. + + Returns: + Number of rules created/updated + """ + try: + # Delete existing rules for this role + existingRules = interface.db.getRecordset(AccessRule, recordFilter={"roleId": roleId}) + for rule in existingRules: + interface.db.recordDelete(AccessRule, rule.get("id")) + + # Create new rules + count = 0 + for ruleData in newRules: + newRule = AccessRule( + roleId=roleId, + context=ruleData.get("context"), + item=ruleData.get("item"), + view=ruleData.get("view", False), + read=ruleData.get("read"), + create=ruleData.get("create"), + update=ruleData.get("update"), + delete=ruleData.get("delete") + ) + interface.db.recordCreate(AccessRule, newRule.model_dump()) + count += 1 + + return count + + except Exception as e: + logger.error(f"Error updating access rules: {e}") + return 0 diff --git a/modules/routes/routeSecurityAdmin.py b/modules/routes/routeSecurityAdmin.py index 19aef5fc..3ba35e65 100644 --- a/modules/routes/routeSecurityAdmin.py +++ b/modules/routes/routeSecurityAdmin.py @@ -1,13 +1,19 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. +""" +Security Administration routes. +MULTI-TENANT: These are SYSTEM-LEVEL operations requiring isSysAdmin=true. +No mandate context - SysAdmin manages infrastructure, not data. +""" from fastapi import APIRouter, HTTPException, Depends, status, Request, Body from fastapi.responses import FileResponse, JSONResponse from typing import Optional, Dict, Any, List import os import logging -from modules.auth import getCurrentUser, limiter -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.auth import getCurrentUser, limiter, requireSysAdmin +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG @@ -26,13 +32,63 @@ router = APIRouter( } ) -def _ensure_admin_scope(current_user: User, target_mandate_id: Optional[str] = None) -> None: - roleLabels = current_user.roleLabels or [] - if "admin" not in roleLabels and "sysadmin" not in roleLabels: - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin privileges required") - if "admin" in roleLabels and "sysadmin" not in roleLabels: - if target_mandate_id and str(target_mandate_id) != str(current_user.mandateId): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Forbidden for target mandate") + +def _getPoweronDatabases() -> List[str]: + """Load databases from PostgreSQL host matching poweron_%.""" + dbHost = APP_CONFIG.get("DB_HOST") + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + # Connect to 'postgres' system database to query all databases + connector = DatabaseConnector( + dbHost=dbHost, + dbDatabase="postgres", + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=None + ) + + try: + with connector.connection.cursor() as cursor: + cursor.execute( + """ + SELECT datname + FROM pg_database + WHERE datname LIKE 'poweron_%' + AND datistemplate = false + ORDER BY datname + """ + ) + rows = cursor.fetchall() + return [row["datname"] for row in rows if row.get("datname")] + finally: + connector.close() + + +def _getDatabaseConnector(databaseName: str, userId: str = None) -> DatabaseConnector: + """ + Create a generic DatabaseConnector for any poweron_* database. + Fully dynamic - no interface mapping needed. + """ + if not databaseName.startswith("poweron_"): + raise ValueError(f"Invalid database name: {databaseName}") + + dbHost = APP_CONFIG.get("DB_HOST") + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + connector = DatabaseConnector( + dbHost=dbHost, + dbDatabase=databaseName, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=userId + ) + return connector # ---------------------- @@ -43,17 +99,19 @@ def _ensure_admin_scope(current_user: User, target_mandate_id: Optional[str] = N @limiter.limit("30/minute") async def list_tokens( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), userId: Optional[str] = None, authority: Optional[str] = None, sessionId: Optional[str] = None, statusFilter: Optional[str] = None, connectionId: Optional[str] = None, ) -> List[Dict[str, Any]]: + """ + List all tokens in the system. + MULTI-TENANT: SysAdmin-only, no mandate filter (system-level view). + """ try: appInterface = getRootInterface() - target_mandate = currentUser.mandateId - _ensure_admin_scope(currentUser, target_mandate) recordFilter: Dict[str, Any] = {} if userId: @@ -66,9 +124,7 @@ async def list_tokens( recordFilter["connectionId"] = connectionId if statusFilter: recordFilter["status"] = statusFilter - roleLabels = currentUser.roleLabels or [] - if "admin" in roleLabels and "sysadmin" not in roleLabels: - recordFilter["mandateId"] = str(currentUser.mandateId) + # MULTI-TENANT: SysAdmin sees ALL tokens (no mandate filter) tokens = appInterface.db.getRecordset(Token, recordFilter=recordFilter) return tokens @@ -83,27 +139,26 @@ async def list_tokens( @limiter.limit("30/minute") async def revoke_tokens_by_user( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: + """ + Revoke all tokens for a user. + MULTI-TENANT: SysAdmin-only, can revoke across all mandates. + """ try: userId = payload.get("userId") authority = payload.get("authority") - reason = payload.get("reason", "admin revoke") + reason = payload.get("reason", "sysadmin revoke") if not userId: raise HTTPException(status_code=400, detail="userId is required") appInterface = getRootInterface() - # Tenant scope check - target_user = appInterface.db.getRecordset(User, recordFilter={"id": userId}) - target_mandate = target_user[0].get("mandateId") if target_user else None - _ensure_admin_scope(currentUser, target_mandate) - - roleLabels = currentUser.roleLabels or [] + # MULTI-TENANT: SysAdmin can revoke any user's tokens (no mandate restriction) count = appInterface.revokeTokensByUser( userId=userId, authority=AuthAuthority(authority) if authority else None, - mandateId=None if "sysadmin" in roleLabels else str(currentUser.mandateId), + mandateId=None, # SysAdmin: no mandate filter revokedBy=currentUser.id, reason=reason ) @@ -119,22 +174,23 @@ async def revoke_tokens_by_user( @limiter.limit("30/minute") async def revoke_tokens_by_session( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: + """ + Revoke all tokens for a specific session. + MULTI-TENANT: SysAdmin-only. + """ try: userId = payload.get("userId") sessionId = payload.get("sessionId") authority = payload.get("authority", "local") - reason = payload.get("reason", "admin session revoke") + reason = payload.get("reason", "sysadmin session revoke") if not userId or not sessionId: raise HTTPException(status_code=400, detail="userId and sessionId are required") appInterface = getRootInterface() - target_user = appInterface.db.getRecordset(User, recordFilter={"id": userId}) - target_mandate = target_user[0].get("mandateId") if target_user else None - _ensure_admin_scope(currentUser, target_mandate) - + # MULTI-TENANT: SysAdmin can revoke any session (no mandate check) count = appInterface.revokeTokensBySessionId( sessionId=sessionId, userId=userId, @@ -154,22 +210,20 @@ async def revoke_tokens_by_session( @limiter.limit("30/minute") async def revoke_token_by_id( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: + """ + Revoke a specific token by ID. + MULTI-TENANT: SysAdmin-only. + """ try: tokenId = payload.get("tokenId") - reason = payload.get("reason", "admin revoke") + reason = payload.get("reason", "sysadmin revoke") if not tokenId: raise HTTPException(status_code=400, detail="tokenId is required") appInterface = getRootInterface() - # Load token to check tenant scope for admins - tokens = appInterface.db.getRecordset(Token, recordFilter={"id": tokenId}) - if not tokens: - return {"revoked": 0} - target_mandate = tokens[0].get("mandateId") - _ensure_admin_scope(currentUser, target_mandate) - + # MULTI-TENANT: SysAdmin can revoke any token (no mandate check) ok = appInterface.revokeTokenById(tokenId, revokedBy=currentUser.id, reason=reason) return {"revoked": 1 if ok else 0} except HTTPException: @@ -183,29 +237,34 @@ async def revoke_token_by_id( @limiter.limit("10/minute") async def revoke_tokens_by_mandate( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: + """ + Revoke all tokens for users in a mandate. + MULTI-TENANT: SysAdmin-only, can revoke tokens for any mandate. + """ try: mandateId = payload.get("mandateId") authority = payload.get("authority", "local") - reason = payload.get("reason", "admin mandate revoke") + reason = payload.get("reason", "sysadmin mandate revoke") if not mandateId: raise HTTPException(status_code=400, detail="mandateId is required") - _ensure_admin_scope(currentUser, mandateId) - - # Revoke for all users in mandate + # MULTI-TENANT: SysAdmin can revoke tokens for any mandate appInterface = getRootInterface() - # IMPORTANT: user rows are stored as UserInDB in the database - users = appInterface.db.getRecordset(UserInDB, recordFilter={"mandateId": mandateId}) + + # Get all UserMandate entries for this mandate to find users + # Note: In new model, users are linked via UserMandate, not User.mandateId + from modules.datamodels.datamodelMembership import UserMandate + userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) + total = 0 - for u in users: - # Revoke regardless of token.mandateId to also catch legacy tokens without mandateId + for um in userMandates: total += appInterface.revokeTokensByUser( - userId=u["id"], - authority=AuthAuthority(authority), - mandateId=None, + userId=um["userId"], + authority=AuthAuthority(authority) if authority else None, + mandateId=None, # Revoke all tokens for user revokedBy=currentUser.id, reason=reason ) @@ -225,10 +284,13 @@ async def revoke_tokens_by_mandate( @limiter.limit("60/minute") async def download_log( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), log_name: str = "poweron" ): - _ensure_admin_scope(currentUser) + """ + Download server logs. + MULTI-TENANT: SysAdmin-only (infrastructure management). + """ base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # base_dir -> gateway if log_name == "poweron": @@ -251,33 +313,18 @@ async def download_log( @limiter.limit("10/minute") async def list_databases( request: Request, - currentUser: User = Depends(getCurrentUser) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: - _ensure_admin_scope(currentUser) - - # Get database names from configuration for each interface - databases = [] - - # App database (interfaceDbAppObjects.py) - app_db = APP_CONFIG.get("DB_APP_DATABASE") - if app_db: - databases.append(app_db) - - # Chat database (interfaceDbChatObjects.py) - chat_db = APP_CONFIG.get("DB_CHAT_DATABASE") - if chat_db: - databases.append(chat_db) - - # Management database (interfaceDbComponentObjects.py) - management_db = APP_CONFIG.get("DB_MANAGEMENT_DATABASE") - if management_db: - databases.append(management_db) - - # Fallback to default if no databases configured - if not databases: - databases = ["poweron"] - - return {"databases": databases} + """ + List all poweron_* databases. + MULTI-TENANT: SysAdmin-only (infrastructure management). + """ + try: + databases = _getPoweronDatabases() + return {"databases": databases} + except Exception as e: + logger.error(f"Failed to load databases from host: {e}") + raise HTTPException(status_code=500, detail="Failed to load databases from host") @router.get("/databases/{database_name}/tables") @@ -285,48 +332,28 @@ async def list_databases( async def get_database_tables( request: Request, database_name: str, - currentUser: User = Depends(getCurrentUser) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: - _ensure_admin_scope(currentUser) - - # Get all configured database names - configured_dbs = [] - app_db = APP_CONFIG.get("DB_APP_DATABASE") - if app_db: - configured_dbs.append(app_db) - chat_db = APP_CONFIG.get("DB_CHAT_DATABASE") - if chat_db: - configured_dbs.append(chat_db) - management_db = APP_CONFIG.get("DB_MANAGEMENT_DATABASE") - if management_db: - configured_dbs.append(management_db) - - if not configured_dbs: - configured_dbs = ["poweron"] - - if database_name not in configured_dbs: - raise HTTPException(status_code=400, detail=f"Invalid database name. Available databases: {configured_dbs}") + """ + List tables in a database. + MULTI-TENANT: SysAdmin-only (infrastructure management). + """ + if not database_name.startswith("poweron_"): + raise HTTPException(status_code=400, detail="Invalid database name format") + connector = None try: - # Use the appropriate interface based on database name - if database_name == app_db: - appInterface = getRootInterface() - tables = appInterface.db.getTables() - elif database_name == chat_db: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - chatInterface = getChatInterface(currentUser) - tables = chatInterface.db.getTables() - elif database_name == management_db: - from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface - componentInterface = getComponentInterface(currentUser) - tables = componentInterface.db.getTables() - else: - raise HTTPException(status_code=400, detail="Database not found") - + connector = _getDatabaseConnector(database_name, currentUser.id) + tables = connector.getTables() return {"tables": tables} + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error getting database tables: {str(e)}") - raise HTTPException(status_code=500, detail="Failed to get database tables") + raise HTTPException(status_code=500, detail=f"Failed to get database tables: {str(e)}") + finally: + if connector: + connector.close() @router.post("/databases/{database_name}/tables/{table_name}/drop") @@ -335,43 +362,20 @@ async def drop_table( request: Request, database_name: str, table_name: str, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: - _ensure_admin_scope(currentUser) - - # Get all configured database names - configured_dbs = [] - app_db = APP_CONFIG.get("DB_APP_DATABASE") - if app_db: - configured_dbs.append(app_db) - chat_db = APP_CONFIG.get("DB_CHAT_DATABASE") - if chat_db: - configured_dbs.append(chat_db) - management_db = APP_CONFIG.get("DB_MANAGEMENT_DATABASE") - if management_db: - configured_dbs.append(management_db) - - if not configured_dbs: - configured_dbs = ["poweron"] - - if database_name not in configured_dbs: - raise HTTPException(status_code=400, detail=f"Invalid database name. Available databases: {configured_dbs}") + """ + Drop a table from a database. + MULTI-TENANT: SysAdmin-only (infrastructure management). + """ + if not database_name.startswith("poweron_"): + raise HTTPException(status_code=400, detail="Invalid database name format") + connector = None try: - # Use the appropriate interface based on database name - if database_name == app_db: - interface = getRootInterface() - elif database_name == chat_db: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - interface = getChatInterface(currentUser) - elif database_name == management_db: - from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface - interface = getComponentInterface(currentUser) - else: - raise HTTPException(status_code=400, detail="Database not found") - - conn = interface.db.connection + connector = _getDatabaseConnector(database_name, currentUser.id) + conn = connector.connection with conn.cursor() as cursor: # Check if table exists cursor.execute(""" @@ -388,57 +392,50 @@ async def drop_table( return {"message": f"Table '{table_name}' dropped successfully from database '{database_name}'"} except HTTPException: raise + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error dropping table: {str(e)}") - if 'interface' in locals() and interface and interface.db and interface.db.connection: - interface.db.connection.rollback() + if connector and connector.connection: + connector.connection.rollback() raise HTTPException(status_code=500, detail="Failed to drop table") + finally: + if connector: + connector.close() @router.post("/databases/drop") @limiter.limit("5/minute") async def drop_database( request: Request, - currentUser: User = Depends(getCurrentUser), + currentUser: User = Depends(requireSysAdmin), payload: Dict[str, Any] = Body(...) ) -> Dict[str, Any]: - _ensure_admin_scope(currentUser) - db_name = payload.get("database") + """ + Drop all tables in a database. + MULTI-TENANT: SysAdmin-only (infrastructure management). + """ + dbName = payload.get("database") - # Get all configured database names - configured_dbs = [] - app_db = APP_CONFIG.get("DB_APP_DATABASE") - if app_db: - configured_dbs.append(app_db) - chat_db = APP_CONFIG.get("DB_CHAT_DATABASE") - if chat_db: - configured_dbs.append(chat_db) - management_db = APP_CONFIG.get("DB_MANAGEMENT_DATABASE") - if management_db: - configured_dbs.append(management_db) + if not dbName or not dbName.startswith("poweron_"): + raise HTTPException(status_code=400, detail="Invalid database name") - if not configured_dbs: - configured_dbs = ["poweron"] - - if not db_name or db_name not in configured_dbs: - raise HTTPException(status_code=400, detail=f"Invalid database name. Available databases: {configured_dbs}") - + # Validate database exists try: - # Use the appropriate interface based on database name - if db_name == app_db: - interface = getRootInterface() - elif db_name == chat_db: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - interface = getChatInterface(currentUser) - elif db_name == management_db: - from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface - interface = getComponentInterface(currentUser) - else: - raise HTTPException(status_code=400, detail="Database not found") - - conn = interface.db.connection + configuredDbs = _getPoweronDatabases() + except Exception as e: + logger.warning(f"Failed to load databases from host: {e}") + configuredDbs = [] + + if configuredDbs and dbName not in configuredDbs: + raise HTTPException(status_code=400, detail=f"Database not found. Available: {configuredDbs}") + + connector = None + try: + connector = _getDatabaseConnector(dbName, currentUser.id) + conn = connector.connection with conn.cursor() as cursor: - # Drop all user tables (public schema) except system table + # Drop all user tables (public schema) cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE' @@ -449,12 +446,17 @@ async def drop_database( cursor.execute(f'DROP TABLE IF EXISTS "{tbl}" CASCADE') dropped.append(tbl) conn.commit() - logger.warning(f"Admin drop_database executed by {currentUser.id}: dropped tables from '{db_name}': {dropped}") + logger.warning(f"Admin drop_database executed by {currentUser.id}: dropped tables from '{dbName}': {dropped}") return {"droppedTables": dropped} + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Error dropping database tables: {str(e)}") - if 'interface' in locals() and interface and interface.db and interface.db.connection: - interface.db.connection.rollback() + if connector and connector.connection: + connector.connection.rollback() raise HTTPException(status_code=500, detail="Failed to drop database tables") + finally: + if connector: + connector.close() diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index fb94555c..4e61a045 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -340,11 +340,12 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo ) # Create JWT token data (like Microsoft does) + # MULTI-TENANT: Token does NOT contain mandateId anymore jwt_token_data = { "sub": user.username, - "mandateId": str(user.mandateId), "userId": str(user.id), "authenticationAuthority": AuthAuthority.GOOGLE.value + # NO mandateId in token - stateless multi-tenant design } # Create JWT access token @@ -360,6 +361,7 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo jti = payload.get("jti") # Create JWT token with matching id + # MULTI-TENANT: Token model no longer has mandateId field token = Token( id=jti, userId=user.id, # Use local user's ID @@ -368,8 +370,8 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo tokenRefresh=token_response.get("refresh_token", ""), tokenType="bearer", expiresAt=jwt_expires_at.timestamp(), - createdAt=getUtcTimestamp(), - mandateId=str(user.mandateId) + createdAt=getUtcTimestamp() + # NO mandateId - Token is not mandate-bound ) # Save access token (no connectionId) @@ -615,11 +617,12 @@ async def logout( appInterface.logout() # Log successful logout + # MULTI-TENANT: Logout is a system-level function, no mandate context try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + mandateId="system", action="logout", successInfo="google_auth_logout" ) diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 89351825..4b64b671 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -83,11 +83,13 @@ async def login( ) # Create token data + # MULTI-TENANT: Token does NOT contain mandateId anymore + # Mandate context is determined per request via X-Mandate-Id header token_data = { "sub": user.username, - "mandateId": str(user.mandateId), "userId": str(user.id), "authenticationAuthority": AuthAuthority.LOCAL + # NO mandateId in token - stateless multi-tenant design } # Create session id and include in token claims for session-scoped logout @@ -116,7 +118,8 @@ async def login( # Get jti from already decoded payload jti = payload.get("jti") - # Create token + # Create token record in database + # MULTI-TENANT: Token model no longer has mandateId field token = Token( id=jti, userId=user.id, @@ -124,19 +127,20 @@ async def login( tokenAccess=access_token, tokenType="bearer", expiresAt=expires_at.timestamp(), - sessionId=session_id, - mandateId=str(user.mandateId) + sessionId=session_id + # NO mandateId - Token is not mandate-bound ) # Save access token userInterface.saveAccessToken(token) # Log successful login + # MULTI-TENANT: Login is a system-level function, no mandate context try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( userId=str(user.id), - mandateId=str(user.mandateId), + mandateId="system", action="login", successInfo="local_auth_success" ) @@ -236,7 +240,6 @@ async def register_user( fullName=userData.fullName, language=userData.language, enabled=True, # Users are enabled by default (can login after setting password) - roleLabels=["user"], # Default role for new registrations authenticationAuthority=AuthAuthority.LOCAL ) @@ -358,11 +361,12 @@ async def refresh_token( raise HTTPException(status_code=500, detail="Failed to validate user") # Create new token data + # MULTI-TENANT: Token does NOT contain mandateId anymore token_data = { "sub": current_user.username, - "mandateId": str(current_user.mandateId), "userId": str(current_user.id), "authenticationAuthority": current_user.authenticationAuthority + # NO mandateId in token } # Create new access token + set cookie @@ -427,11 +431,12 @@ async def logout(request: Request, response: Response, currentUser: User = Depen revoked = 1 # Log successful logout + # MULTI-TENANT: Logout is a system-level function, no mandate context try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + mandateId="system", action="logout", successInfo=f"revoked_tokens: {revoked}" ) diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 4b2964db..c145d1d3 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -348,11 +348,12 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo appInterface.saveAccessToken(token) # Create JWT token data + # MULTI-TENANT: Token does NOT contain mandateId anymore jwt_token_data = { "sub": user.username, - "mandateId": str(user.mandateId), "userId": str(user.id), "authenticationAuthority": AuthAuthority.MSFT.value + # NO mandateId in token - stateless multi-tenant design } # Create JWT access token @@ -368,6 +369,7 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo jti = payload.get("jti") # Create JWT token with matching id + # MULTI-TENANT: Token model no longer has mandateId field jwt_token_obj = Token( id=jti, userId=user.id, @@ -375,8 +377,8 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo tokenAccess=jwt_token, tokenType="bearer", expiresAt=jwt_expires_at.timestamp(), - createdAt=getUtcTimestamp(), - mandateId=str(user.mandateId) + createdAt=getUtcTimestamp() + # NO mandateId - Token is not mandate-bound ) # Save JWT access token @@ -625,11 +627,12 @@ async def logout( appInterface.logout() # Log successful logout + # MULTI-TENANT: Logout is a system-level function, no mandate context try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( userId=str(currentUser.id), - mandateId=str(currentUser.mandateId), + mandateId="system", action="logout", successInfo="microsoft_auth_logout" ) diff --git a/modules/security/rbac.py b/modules/security/rbac.py index c8a58e92..f236852a 100644 --- a/modules/security/rbac.py +++ b/modules/security/rbac.py @@ -2,14 +2,24 @@ # All rights reserved. """ RBAC interface: Core RBAC logic and permission resolution. -Moved from interfaces to security module to maintain proper architectural layering. -Connectors can import from security, but not from interfaces. + +Multi-Tenant Design: +- AccessRules referenzieren roleId (FK), nicht roleLabel +- Rollen werden über UserMandate + UserMandateRole geladen +- Priorisierung: Instance > Mandate > Global +- Stateless Design: Kein Cache, direkt aus DB """ import logging -from typing import List, Optional, Dict, Any, TYPE_CHECKING -from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext +from typing import List, Optional, TYPE_CHECKING +from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel +from modules.datamodels.datamodelMembership import ( + UserMandate, + UserMandateRole, + FeatureAccess, + FeatureAccessRole +) if TYPE_CHECKING: from modules.connectors.connectorDbPostgre import DatabaseConnector @@ -20,6 +30,11 @@ logger = logging.getLogger(__name__) class RbacClass: """ RBAC interface for permission resolution and rule validation. + + Multi-Tenant Design: + - Lädt Rollen über UserMandate + UserMandateRole + - AccessRules werden über roleId gefunden + - isSysAdmin für System-Level Operationen (ohne Mandant) """ def __init__(self, db: "DatabaseConnector", dbApp: "DatabaseConnector"): @@ -34,14 +49,27 @@ class RbacClass: self.db = db self.dbApp = dbApp - def getUserPermissions(self, user: User, context: AccessRuleContext, item: str) -> UserPermissions: + def getUserPermissions( + self, + user: User, + context: AccessRuleContext, + item: str, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ) -> UserPermissions: """ Get combined permissions for a user across all their roles. + Multi-Tenant Design: + - Lädt Rollen aus UserMandate + UserMandateRole wenn mandateId gegeben + - isSysAdmin gibt vollen Zugriff auf System-Level (kein mandateId) + Args: - user: User object with roleLabels + user: User object context: Access rule context (DATA, UI, RESOURCE) item: Item identifier (table name, UI path, resource path) + mandateId: Optional mandate context for role lookup + featureInstanceId: Optional feature instance context Returns: UserPermissions object with combined permissions @@ -54,23 +82,37 @@ class RbacClass: delete=AccessLevel.NONE ) - if not hasattr(user, 'roleLabels') or not user.roleLabels: + # SysAdmin auf System-Level (kein Mandant) hat vollen Zugriff + if hasattr(user, 'isSysAdmin') and user.isSysAdmin and not mandateId: + return UserPermissions( + view=True, + read=AccessLevel.ALL, + create=AccessLevel.ALL, + update=AccessLevel.ALL, + delete=AccessLevel.ALL + ) + + # Lade Role-IDs für den User via UserMandate + UserMandateRole + roleIds = self._getRoleIdsForUser(user, mandateId, featureInstanceId) + + if not roleIds: return permissions - # Step 1: For each role, find the most specific matching rule (most specific wins within role) - rolePermissions = {} - for roleLabel in user.roleLabels: - # Get all rules for this role and context - allRules = self._getRulesForRole(roleLabel, context) - - # Find most specific rule for this item (longest matching prefix) - mostSpecificRule = self.findMostSpecificRule(allRules, item) - - if mostSpecificRule: - rolePermissions[roleLabel] = mostSpecificRule + # Lade alle relevanten Regeln für alle Rollen + allRulesWithPriority = self._getRulesForRoleIds(roleIds, context, mandateId, featureInstanceId) - # Step 2: Combine permissions across roles using opening (union) logic - for roleLabel, rule in rolePermissions.items(): + # Für jede Rolle die spezifischste Regel finden + rolePermissions = {} + for priority, rule in allRulesWithPriority: + # Find most specific rule for this item + if self._ruleMatchesItem(rule, item): + roleId = rule.roleId + # Speichere mit Priorität (höhere Priorität überschreibt) + if roleId not in rolePermissions or priority > rolePermissions[roleId][0]: + rolePermissions[roleId] = (priority, rule) + + # Combine permissions across roles using opening (union) logic + for roleId, (priority, rule) in rolePermissions.items(): # View: union logic - if ANY role has view=true, then view=true if rule.view: permissions.view = True @@ -88,6 +130,274 @@ class RbacClass: return permissions + def _getRoleIdsForUser( + self, + user: User, + mandateId: Optional[str], + featureInstanceId: Optional[str] + ) -> List[str]: + """ + Get all role IDs for a user in the given context. + Uses UserMandate + UserMandateRole for the new multi-tenant model. + + Args: + user: User object + mandateId: Mandate context + featureInstanceId: Feature instance context + + Returns: + List of role IDs + """ + roleIds = [] + + if not mandateId: + return roleIds + + try: + # Lade UserMandate + userMandates = self.dbApp.getRecordset( + UserMandate, + recordFilter={"userId": user.id, "mandateId": mandateId, "enabled": True} + ) + + if not userMandates: + return roleIds + + userMandateId = userMandates[0].get("id") + + # Lade UserMandateRoles (Mandate-level roles) + userMandateRoles = self.dbApp.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId} + ) + + roleIds.extend([r.get("roleId") for r in userMandateRoles if r.get("roleId")]) + + # Load FeatureAccess + FeatureAccessRole (Instance-level roles) + if featureInstanceId: + featureAccessRecords = self.dbApp.getRecordset( + FeatureAccess, + recordFilter={ + "userId": user.id, + "featureInstanceId": featureInstanceId, + "enabled": True + } + ) + + if featureAccessRecords: + featureAccessId = featureAccessRecords[0].get("id") + + featureAccessRoles = self.dbApp.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + + roleIds.extend([r.get("roleId") for r in featureAccessRoles if r.get("roleId")]) + + except Exception as e: + logger.error(f"Error loading role IDs for user {user.id}: {e}") + + return roleIds + + def getRulesForUserBulk( + self, + userId: str, + mandateId: str, + featureInstanceId: Optional[str] = None + ) -> List[tuple]: + """ + Lädt alle relevanten Regeln für einen User in EINEM Query. + Stateless: Kein Cache, direkt aus DB. + + Optimiert für Multi-Tenant mit Junction Tables: + - Mandant-Rollen via UserMandate → UserMandateRole + - Instanz-Rollen via FeatureAccess → FeatureAccessRole + + Args: + userId: User ID + mandateId: Mandate context + featureInstanceId: Optional feature instance context + + Returns: + Liste von (priority, AccessRule) Tupeln + """ + if not mandateId: + return [] + + try: + conn = self.dbApp.connection + roleIds = set() + + # 1. Mandant-Rollen via UserMandate → UserMandateRole (SINGLE Query) + with conn.cursor() as cursor: + cursor.execute( + """ + SELECT umr."roleId" + FROM "UserMandate" um + JOIN "UserMandateRole" umr ON umr."userMandateId" = um.id + WHERE um."userId" = %s AND um."mandateId" = %s AND um."enabled" = true + """, + (userId, mandateId) + ) + mandateRoles = cursor.fetchall() + roleIds.update(r["roleId"] for r in mandateRoles if r.get("roleId")) + + # 2. Instanz-Rollen via FeatureAccess → FeatureAccessRole (SINGLE Query) + if featureInstanceId: + with conn.cursor() as cursor: + cursor.execute( + """ + SELECT far."roleId" + FROM "FeatureAccess" fa + JOIN "FeatureAccessRole" far ON far."featureAccessId" = fa.id + WHERE fa."userId" = %s AND fa."featureInstanceId" = %s AND fa."enabled" = true + """, + (userId, featureInstanceId) + ) + instanceRoles = cursor.fetchall() + roleIds.update(r["roleId"] for r in instanceRoles if r.get("roleId")) + + if not roleIds: + return [] + + # 3. BULK Query: Alle Regeln für alle Rollen + zugehörige Role-Daten + # SINGLE Query mit JOIN statt N+1 + roleIdsList = list(roleIds) + with conn.cursor() as cursor: + cursor.execute( + """ + SELECT ar.*, r."mandateId" as "roleMandateId", + r."featureInstanceId" as "roleInstanceId" + FROM "AccessRule" ar + JOIN "Role" r ON ar."roleId" = r.id + WHERE ar."roleId" = ANY(%s) + """, + (roleIdsList,) + ) + allRulesWithContext = cursor.fetchall() + + # 4. Priorität zuweisen basierend auf Role-Scope + rulesWithPriority = [] + for ruleRecord in allRulesWithContext: + ruleDict = dict(ruleRecord) + + # Bestimme Priorität + if ruleDict.get("roleInstanceId"): + priority = 3 # Instance-Rolle = höchste Priorität + elif ruleDict.get("roleMandateId"): + priority = 2 # Mandate-Rolle + else: + priority = 1 # Global-Rolle = niedrigste Priorität + + # Entferne Hilfsspalten vor AccessRule-Erstellung + ruleDict.pop("roleMandateId", None) + ruleDict.pop("roleInstanceId", None) + + try: + rule = AccessRule(**ruleDict) + rulesWithPriority.append((priority, rule)) + except Exception as e: + logger.error(f"Error converting rule record: {e}") + + return rulesWithPriority + + except Exception as e: + logger.error(f"Error in getRulesForUserBulk: {e}") + return [] + + def _getRulesForRoleIds( + self, + roleIds: List[str], + context: AccessRuleContext, + mandateId: Optional[str], + featureInstanceId: Optional[str] + ) -> List[tuple]: + """ + Get all access rules for the given role IDs with priority. + + Priority: + - 3: Instance-specific role (featureInstanceId set) + - 2: Mandate-specific role (mandateId set, no featureInstanceId) + - 1: Global role (no mandateId) + + Args: + roleIds: List of role IDs + context: Access rule context + mandateId: Current mandate context + featureInstanceId: Current feature instance context + + Returns: + List of (priority, AccessRule) tuples + """ + rulesWithPriority = [] + + if not roleIds: + return rulesWithPriority + + try: + # Lade alle Regeln für alle Rollen + for roleId in roleIds: + rules = self.dbApp.getRecordset( + AccessRule, + recordFilter={"roleId": roleId, "context": context.value} + ) + + # Lade Role um Priorität zu bestimmen + roleRecords = self.dbApp.getRecordset(Role, recordFilter={"id": roleId}) + if not roleRecords: + continue + + role = roleRecords[0] + + # Bestimme Priorität basierend auf Role-Scope + if role.get("featureInstanceId"): + priority = 3 # Instance-specific + elif role.get("mandateId"): + priority = 2 # Mandate-specific + else: + priority = 1 # Global + + for ruleRecord in rules: + try: + rule = AccessRule(**ruleRecord) + rulesWithPriority.append((priority, rule)) + except Exception as e: + logger.error(f"Error converting rule record: {e}") + + except Exception as e: + logger.error(f"Error loading rules for role IDs: {e}") + + return rulesWithPriority + + def _ruleMatchesItem(self, rule: AccessRule, item: str) -> bool: + """ + Check if a rule matches the given item. + + Args: + rule: Access rule to check + item: Item to match against + + Returns: + True if rule matches item + """ + if rule.item is None: + # Generic rule matches everything + return True + + if not item: + # No item specified, only generic rules match + return rule.item is None + + # Exact match + if rule.item == item: + return True + + # Prefix match (e.g., "trustee" matches "trustee.contract") + if item.startswith(rule.item + "."): + return True + + return False + def findMostSpecificRule(self, rules: List[AccessRule], item: str) -> Optional[AccessRule]: """ Find the most specific rule for an item (longest matching prefix wins). @@ -105,7 +415,6 @@ class RbacClass: return genericRules[0] if genericRules else None # Find longest matching prefix - itemParts = item.split(".") bestMatch = None bestMatchLength = -1 @@ -176,39 +485,3 @@ class RbacClass: AccessLevel.ALL: 3 } return hierarchy.get(level1, 0) > hierarchy.get(level2, 0) - - def _getRulesForRole(self, roleLabel: str, context: AccessRuleContext) -> List[AccessRule]: - """ - Get all access rules for a specific role and context. - Always queries from DbApp database, not the current database. - - Args: - roleLabel: Role label to get rules for - context: Context type - - Returns: - List of AccessRule objects - """ - try: - # Always use DbApp database for AccessRule queries - rules = self.dbApp.getRecordset( - AccessRule, - recordFilter={ - "roleLabel": roleLabel, - "context": context.value - } - ) - - # Convert dict records to AccessRule objects - accessRules = [] - for record in rules: - try: - accessRule = AccessRule(**record) - accessRules.append(accessRule) - except Exception as e: - logger.error(f"Error converting rule record to AccessRule: {e}, record={record}") - - return accessRules - except Exception as e: - logger.error(f"Error getting rules for role {roleLabel} and context {context.value}: {e}", exc_info=True) - return [] diff --git a/modules/security/rootAccess.py b/modules/security/rootAccess.py index 2d4a2ae2..d87c22e8 100644 --- a/modules/security/rootAccess.py +++ b/modules/security/rootAccess.py @@ -3,6 +3,8 @@ """ Root access management for system-level operations. Provides secure access to root user and DbApp database connector. + +Bei leerer Datenbank wird automatisch Bootstrap ausgeführt. """ import logging @@ -14,6 +16,7 @@ logger = logging.getLogger(__name__) _rootDbAppConnector = None _rootUser = None +_bootstrapExecuted = False def getRootDbAppConnector() -> DatabaseConnector: """ @@ -24,29 +27,62 @@ def getRootDbAppConnector() -> DatabaseConnector: if _rootDbAppConnector is None: _rootDbAppConnector = DatabaseConnector( - dbHost=APP_CONFIG.get("DB_APP_HOST"), - dbDatabase=APP_CONFIG.get("DB_APP_DATABASE", "app"), - dbUser=APP_CONFIG.get("DB_APP_USER"), - dbPassword=APP_CONFIG.get("DB_APP_PASSWORD_SECRET"), - dbPort=int(APP_CONFIG.get("DB_APP_PORT", 5432)), + dbHost=APP_CONFIG.get("DB_HOST"), + dbDatabase="poweron_app", + dbUser=APP_CONFIG.get("DB_USER"), + dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"), + dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), userId=None # No user context for root connector ) _rootDbAppConnector.initDbSystem() return _rootDbAppConnector + +def _ensureBootstrap(): + """ + Führt Bootstrap aus, falls noch nicht geschehen. + Wird automatisch aufgerufen, wenn getRootUser() keinen User findet. + """ + global _bootstrapExecuted + + if _bootstrapExecuted: + return + + logger.info("Running bootstrap to initialize database") + + # Import here to avoid circular imports + from modules.interfaces.interfaceBootstrap import initBootstrap + + dbApp = getRootDbAppConnector() + initBootstrap(dbApp) + + _bootstrapExecuted = True + logger.info("Bootstrap completed") + + def getRootUser() -> User: """ Returns the root user (initial user from database). Used for system-level operations that require root privileges. + + Falls kein User existiert, wird Bootstrap automatisch ausgeführt. """ global _rootUser if _rootUser is None: dbApp = getRootDbAppConnector() initialUserId = dbApp.getInitialId(UserInDB) + + # Wenn kein User existiert, Bootstrap ausführen if not initialUserId: - raise ValueError("No initial user ID found in database") + logger.info("No initial user found, running bootstrap") + _ensureBootstrap() + + # Nochmal versuchen nach Bootstrap + initialUserId = dbApp.getInitialId(UserInDB) + if not initialUserId: + raise ValueError("No initial user ID found in database after bootstrap") users = dbApp.getRecordset(UserInDB, recordFilter={"id": initialUserId}) if not users: @@ -56,4 +92,3 @@ def getRootUser() -> User: _rootUser = User(**user_data) return _rootUser - diff --git a/modules/services/__init__.py b/modules/services/__init__.py index 16d2ed6d..7033bfbb 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -from typing import Any +from typing import Any, Optional from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelChat import ChatWorkflow @@ -40,25 +40,26 @@ class PublicService: class Services: - def __init__(self, user: User, workflow: ChatWorkflow = None): + def __init__(self, user: User, workflow: ChatWorkflow = None, mandateId: Optional[str] = None): self.user: User = user self.workflow: ChatWorkflow = workflow + self.mandateId: Optional[str] = mandateId self.currentUserPrompt: str = "" # Cleaned/normalized user intent for the current round self.rawUserPrompt: str = "" # Original raw user message for the current round - # Initialize interfaces + # Initialize interfaces with explicit mandateId from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - self.interfaceDbChat = getChatInterface(user) + self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface - self.interfaceDbApp = getAppInterface(user) + self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface - self.interfaceDbComponent = getComponentInterface(user) + self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbTrusteeObjects import getInterface as getTrusteeInterface - self.interfaceDbTrustee = getTrusteeInterface(user) + self.interfaceDbTrustee = getTrusteeInterface(user, mandateId=mandateId) # Expose RBAC directly on services for convenience self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None @@ -99,7 +100,15 @@ class Services: self.messaging = PublicService(MessagingService(self)) -def getInterface(user: User, workflow: ChatWorkflow) -> Services: - return Services(user, workflow) +def getInterface(user: User, workflow: ChatWorkflow = None, mandateId: Optional[str] = None) -> Services: + """ + Get Services instance for the given user and mandate context. + + Args: + user: The authenticated user + workflow: Optional ChatWorkflow context + mandateId: Explicit mandate context (from RequestContext / X-Mandate-Id header). Required. + """ + return Services(user, workflow, mandateId=mandateId) diff --git a/modules/shared/dbMultiTenantOptimizations.py b/modules/shared/dbMultiTenantOptimizations.py new file mode 100644 index 00000000..38f154a5 --- /dev/null +++ b/modules/shared/dbMultiTenantOptimizations.py @@ -0,0 +1,438 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Database optimizations for Multi-Tenant model. + +Applies indexes, immutable triggers, and foreign key constraints +for the junction tables used in the multi-tenant mandate model. + +Usage: + from modules.shared.dbMultiTenantOptimizations import applyMultiTenantOptimizations + + # Call after database tables are created + applyMultiTenantOptimizations(dbConnector) + +All operations are idempotent (safe to call multiple times). +""" + +import logging +from typing import Optional, List + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Index Definitions +# ============================================================================= + +_INDEXES = [ + # UserMandate indexes + ("UserMandate", "idx_usermandate_user", ["userId"]), + ("UserMandate", "idx_usermandate_user_mandate", ["userId", "mandateId"]), + ("UserMandate", "idx_usermandate_mandate", ["mandateId"]), + + # UserMandateRole indexes + ("UserMandateRole", "idx_usermandaterole_usermandate", ["userMandateId"]), + ("UserMandateRole", "idx_usermandaterole_role", ["roleId"]), + + # FeatureAccess indexes + ("FeatureAccess", "idx_featureaccess_user_instance", ["userId", "featureInstanceId"]), + ("FeatureAccess", "idx_featureaccess_user", ["userId"]), + ("FeatureAccess", "idx_featureaccess_instance", ["featureInstanceId"]), + + # FeatureAccessRole indexes + ("FeatureAccessRole", "idx_featureaccessrole_featureaccess", ["featureAccessId"]), + ("FeatureAccessRole", "idx_featureaccessrole_role", ["roleId"]), + + # AccessRule indexes + ("AccessRule", "idx_accessrule_roleid", ["roleId"]), + ("AccessRule", "idx_accessrule_context_roleid", ["context", "roleId"]), + + # Role indexes + ("Role", "idx_role_mandate_instance", ["mandateId", "featureInstanceId"]), + ("Role", "idx_role_label", ["roleLabel"]), + + # FeatureInstance indexes + ("FeatureInstance", "idx_featureinstance_mandate", ["mandateId"]), + ("FeatureInstance", "idx_featureinstance_mandate_code", ["mandateId", "featureCode"]), + + # Invitation indexes + ("Invitation", "idx_invitation_mandate", ["mandateId"]), + ("Invitation", "idx_invitation_createdby", ["createdBy"]), +] + +# Unique indexes (separate list) +_UNIQUE_INDEXES = [ + ("Invitation", "idx_invitation_token", ["token"]), +] + +# Partial indexes (with WHERE clause) +_PARTIAL_INDEXES = [ + ("UserMandate", "idx_usermandate_user_enabled", ["userId"], '"enabled" = true'), + ("Role", "idx_role_featurecode", ["featureCode"], '"mandateId" IS NULL'), +] + + +# ============================================================================= +# Foreign Key Definitions +# ============================================================================= + +_FOREIGN_KEYS = [ + # UserMandate FKs + ("UserMandate", "fk_usermandate_mandate", "mandateId", "Mandate", "id"), + ("UserMandate", "fk_usermandate_user", "userId", "User", "id"), + + # FeatureInstance FKs + ("FeatureInstance", "fk_featureinstance_mandate", "mandateId", "Mandate", "id"), + + # Role FKs (nullable - only cascade when not null) + ("Role", "fk_role_mandate", "mandateId", "Mandate", "id"), + ("Role", "fk_role_instance", "featureInstanceId", "FeatureInstance", "id"), + + # FeatureAccess FKs + ("FeatureAccess", "fk_featureaccess_instance", "featureInstanceId", "FeatureInstance", "id"), + ("FeatureAccess", "fk_featureaccess_user", "userId", "User", "id"), + + # AccessRule FKs + ("AccessRule", "fk_accessrule_role", "roleId", "Role", "id"), + + # Junction table FKs + ("UserMandateRole", "fk_usermandaterole_usermandate", "userMandateId", "UserMandate", "id"), + ("UserMandateRole", "fk_usermandaterole_role", "roleId", "Role", "id"), + ("FeatureAccessRole", "fk_featureaccessrole_featureaccess", "featureAccessId", "FeatureAccess", "id"), + ("FeatureAccessRole", "fk_featureaccessrole_role", "roleId", "Role", "id"), + + # Invitation FKs + ("Invitation", "fk_invitation_mandate", "mandateId", "Mandate", "id"), +] + + +# ============================================================================= +# Immutable Trigger Definitions +# ============================================================================= + +_IMMUTABLE_TRIGGERS = [ + # Role: mandateId, featureInstanceId, featureCode are immutable + ("Role", "tr_role_immutable", ["mandateId", "featureInstanceId", "featureCode"]), + + # AccessRule: context, roleId are immutable + ("AccessRule", "tr_accessrule_immutable", ["context", "roleId"]), +] + + +# ============================================================================= +# Main Functions +# ============================================================================= + +def applyMultiTenantOptimizations(dbConnector, tables: Optional[List[str]] = None) -> dict: + """ + Apply all multi-tenant database optimizations. + + Args: + dbConnector: Database connector with execute capability + tables: Optional list of table names to optimize. If None, optimizes all. + + Returns: + dict with counts of created indexes, triggers, and foreign keys + """ + results = { + "indexesCreated": 0, + "triggersCreated": 0, + "foreignKeysCreated": 0, + "errors": [] + } + + try: + # Get a connection from the connector + conn = dbConnector._get_connection() + conn.autocommit = True + + with conn.cursor() as cursor: + # Apply indexes + results["indexesCreated"] = _applyIndexes(cursor, tables) + + # Apply foreign keys + results["foreignKeysCreated"] = _applyForeignKeys(cursor, tables) + + # Apply immutable triggers + results["triggersCreated"] = _applyImmutableTriggers(cursor, tables) + + logger.info( + f"Multi-tenant optimizations applied: " + f"{results['indexesCreated']} indexes, " + f"{results['triggersCreated']} triggers, " + f"{results['foreignKeysCreated']} foreign keys" + ) + + except Exception as e: + logger.error(f"Error applying multi-tenant optimizations: {e}") + results["errors"].append(str(e)) + + return results + + +def applyIndexesOnly(dbConnector, tables: Optional[List[str]] = None) -> int: + """Apply only indexes (lighter operation, safe for frequent calls).""" + try: + conn = dbConnector._get_connection() + conn.autocommit = True + + with conn.cursor() as cursor: + return _applyIndexes(cursor, tables) + except Exception as e: + logger.error(f"Error applying indexes: {e}") + return 0 + + +# ============================================================================= +# Internal Implementation +# ============================================================================= + +def _tableExists(cursor, tableName: str) -> bool: + """Check if a table exists in the database.""" + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = %s + ) + """, (tableName,)) + return cursor.fetchone()[0] + + +def _indexExists(cursor, indexName: str) -> bool: + """Check if an index exists.""" + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM pg_indexes + WHERE indexname = %s + ) + """, (indexName,)) + return cursor.fetchone()[0] + + +def _constraintExists(cursor, constraintName: str) -> bool: + """Check if a constraint exists.""" + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM pg_constraint + WHERE conname = %s + ) + """, (constraintName,)) + return cursor.fetchone()[0] + + +def _triggerExists(cursor, triggerName: str) -> bool: + """Check if a trigger exists.""" + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM pg_trigger + WHERE tgname = %s + ) + """, (triggerName,)) + return cursor.fetchone()[0] + + +def _applyIndexes(cursor, tables: Optional[List[str]]) -> int: + """Apply all indexes. Returns count of newly created indexes.""" + created = 0 + + # Regular indexes + for tableName, indexName, columns in _INDEXES: + if tables and tableName not in tables: + continue + if not _tableExists(cursor, tableName): + continue + if _indexExists(cursor, indexName): + continue + + try: + columnList = ", ".join(f'"{c}"' for c in columns) + cursor.execute(f'CREATE INDEX "{indexName}" ON "{tableName}" ({columnList})') + created += 1 + logger.debug(f"Created index {indexName} on {tableName}") + except Exception as e: + logger.warning(f"Failed to create index {indexName}: {e}") + + # Unique indexes + for tableName, indexName, columns in _UNIQUE_INDEXES: + if tables and tableName not in tables: + continue + if not _tableExists(cursor, tableName): + continue + if _indexExists(cursor, indexName): + continue + + try: + columnList = ", ".join(f'"{c}"' for c in columns) + cursor.execute(f'CREATE UNIQUE INDEX "{indexName}" ON "{tableName}" ({columnList})') + created += 1 + logger.debug(f"Created unique index {indexName} on {tableName}") + except Exception as e: + logger.warning(f"Failed to create unique index {indexName}: {e}") + + # Partial indexes + for tableName, indexName, columns, whereClause in _PARTIAL_INDEXES: + if tables and tableName not in tables: + continue + if not _tableExists(cursor, tableName): + continue + if _indexExists(cursor, indexName): + continue + + try: + columnList = ", ".join(f'"{c}"' for c in columns) + cursor.execute(f'CREATE INDEX "{indexName}" ON "{tableName}" ({columnList}) WHERE {whereClause}') + created += 1 + logger.debug(f"Created partial index {indexName} on {tableName}") + except Exception as e: + logger.warning(f"Failed to create partial index {indexName}: {e}") + + return created + + +def _applyForeignKeys(cursor, tables: Optional[List[str]]) -> int: + """Apply foreign key constraints with CASCADE DELETE. Returns count created.""" + created = 0 + + for tableName, constraintName, column, refTable, refColumn in _FOREIGN_KEYS: + if tables and tableName not in tables: + continue + if not _tableExists(cursor, tableName): + continue + if not _tableExists(cursor, refTable): + continue + if _constraintExists(cursor, constraintName): + continue + + try: + cursor.execute(f""" + ALTER TABLE "{tableName}" + ADD CONSTRAINT "{constraintName}" + FOREIGN KEY ("{column}") + REFERENCES "{refTable}"("{refColumn}") + ON DELETE CASCADE + """) + created += 1 + logger.debug(f"Created FK {constraintName} on {tableName}") + except Exception as e: + logger.warning(f"Failed to create FK {constraintName}: {e}") + + return created + + +def _applyImmutableTriggers(cursor, tables: Optional[List[str]]) -> int: + """Apply immutable field triggers. Returns count created.""" + created = 0 + + for tableName, triggerName, immutableFields in _IMMUTABLE_TRIGGERS: + if tables and tableName not in tables: + continue + if not _tableExists(cursor, tableName): + continue + if _triggerExists(cursor, triggerName): + continue + + try: + # Create the function + functionName = f"fn_{triggerName}" + checks = [] + for field in immutableFields: + checks.append(f""" + IF OLD."{field}" IS DISTINCT FROM NEW."{field}" THEN + RAISE EXCEPTION '{field} is immutable on {tableName}. Delete and recreate instead.'; + END IF; + """) + + functionBody = "\n".join(checks) + + cursor.execute(f""" + CREATE OR REPLACE FUNCTION "{functionName}"() + RETURNS TRIGGER AS $$ + BEGIN + {functionBody} + RETURN NEW; + END; + $$ LANGUAGE plpgsql + """) + + # Create the trigger + cursor.execute(f""" + CREATE TRIGGER "{triggerName}" + BEFORE UPDATE ON "{tableName}" + FOR EACH ROW + EXECUTE FUNCTION "{functionName}"() + """) + + created += 1 + logger.debug(f"Created immutable trigger {triggerName} on {tableName}") + except Exception as e: + logger.warning(f"Failed to create trigger {triggerName}: {e}") + + return created + + +# ============================================================================= +# Utility: Check optimization status +# ============================================================================= + +def getOptimizationStatus(dbConnector) -> dict: + """ + Check which optimizations are already applied. + + Returns dict with lists of applied and missing optimizations. + """ + status = { + "indexes": {"applied": [], "missing": []}, + "uniqueIndexes": {"applied": [], "missing": []}, + "partialIndexes": {"applied": [], "missing": []}, + "foreignKeys": {"applied": [], "missing": []}, + "triggers": {"applied": [], "missing": []} + } + + try: + conn = dbConnector._get_connection() + with conn.cursor() as cursor: + # Check regular indexes + for tableName, indexName, _ in _INDEXES: + if _tableExists(cursor, tableName): + if _indexExists(cursor, indexName): + status["indexes"]["applied"].append(indexName) + else: + status["indexes"]["missing"].append(indexName) + + # Check unique indexes + for tableName, indexName, _ in _UNIQUE_INDEXES: + if _tableExists(cursor, tableName): + if _indexExists(cursor, indexName): + status["uniqueIndexes"]["applied"].append(indexName) + else: + status["uniqueIndexes"]["missing"].append(indexName) + + # Check partial indexes + for tableName, indexName, _, _ in _PARTIAL_INDEXES: + if _tableExists(cursor, tableName): + if _indexExists(cursor, indexName): + status["partialIndexes"]["applied"].append(indexName) + else: + status["partialIndexes"]["missing"].append(indexName) + + # Check foreign keys + for tableName, constraintName, _, _, _ in _FOREIGN_KEYS: + if _tableExists(cursor, tableName): + if _constraintExists(cursor, constraintName): + status["foreignKeys"]["applied"].append(constraintName) + else: + status["foreignKeys"]["missing"].append(constraintName) + + # Check triggers + for tableName, triggerName, _ in _IMMUTABLE_TRIGGERS: + if _tableExists(cursor, tableName): + if _triggerExists(cursor, triggerName): + status["triggers"]["applied"].append(triggerName) + else: + status["triggers"]["missing"].append(triggerName) + + except Exception as e: + logger.error(f"Error checking optimization status: {e}") + + return status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index 980ce120..a9b656eb 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -96,7 +96,7 @@ class WorkflowManager: "currentAction": 0, "totalTasks": 0, "totalActions": 0, - "mandateId": self.services.user.mandateId, + "mandateId": self.services.mandateId, "messageIds": [], "workflowMode": workflowMode, "maxSteps": 10 , # Set maxSteps From ccc41e70237767aad293a7ea16fa57d1e2144390 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 19 Jan 2026 09:18:37 +0100 Subject: [PATCH 02/32] harmonized module names --- app.py | 6 +- .../migration_export_20260119_085558.json | 1221 +++++++++++++++++ modules/datamodels/datamodelAudit.py | 208 +++ .../{datamodelChat.py => datamodelChatbot.py} | 36 + modules/datamodels/datamodelFiles.py | 2 + modules/datamodels/datamodelMessaging.py | 25 + modules/datamodels/datamodelNeutralizer.py | 4 + modules/datamodels/datamodelRealEstate.py | 40 + modules/datamodels/datamodelTrustee.py | 70 + modules/datamodels/datamodelVoice.py | 2 + modules/datamodels/datamodelWorkflow.py | 2 +- .../datamodels/datamodelWorkflowActions.py | 2 +- modules/features/chatbot/mainChatbot.py | 4 +- modules/features/realEstate/mainRealEstate.py | 2 +- modules/features/workflow/mainWorkflow.py | 2 +- .../features/workflow/subAutomationUtils.py | 2 +- ...DbChatObjects.py => interfaceDbChatbot.py} | 70 +- .../interfaces/interfaceDbComponentObjects.py | 35 +- ...ateObjects.py => interfaceDbRealEstate.py} | 40 +- ...rusteeObjects.py => interfaceDbTrustee.py} | 37 +- modules/routes/routeDataAutomation.py | 4 +- modules/routes/routeDataMandates.py | 28 +- modules/routes/routeDataUsers.py | 8 +- modules/routes/routeFeatureChatDynamic.py | 6 +- modules/routes/routeFeatureChatbot.py | 6 +- modules/routes/routeFeatureRealEstate.py | 2 +- modules/routes/routeFeatureTrustee.py | 364 +++-- ...eAutomation.py => routeFeatureWorkflow.py} | 6 +- modules/routes/routeGdpr.py | 21 +- modules/routes/routeSecurityGoogle.py | 5 +- modules/routes/routeSecurityLocal.py | 21 +- modules/routes/routeSecurityMsft.py | 5 +- modules/routes/routeWorkflows.py | 8 +- modules/services/__init__.py | 6 +- modules/services/serviceAi/mainServiceAi.py | 2 +- .../serviceAi/subContentExtraction.py | 2 +- .../services/serviceAi/subDocumentIntents.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../mainServiceExtraction.py | 2 +- .../mainServiceGeneration.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 6 +- modules/shared/auditLogger.py | 548 ++++++-- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 8 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 12 +- modules/workflows/workflowManager.py | 10 +- tests/functional/test02_ai_models.py | 2 +- tests/functional/test03_ai_operations.py | 18 +- tests/functional/test04_ai_behavior.py | 6 +- .../test05_workflow_with_documents.py | 8 +- .../test06_workflow_prompt_variations.py | 8 +- .../test09_document_generation_formats.py | 8 +- .../test10_document_generation_formats.py | 8 +- .../test11_code_generation_formats.py | 8 +- .../workflows/test_workflow_execution.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- tool_db_export_migration.py | 508 +++++++ tool_db_import_migration.py | 612 +++++++++ 100 files changed, 3756 insertions(+), 432 deletions(-) create mode 100644 local/backup/migration_export_20260119_085558.json create mode 100644 modules/datamodels/datamodelAudit.py rename modules/datamodels/{datamodelChat.py => datamodelChatbot.py} (95%) rename modules/interfaces/{interfaceDbChatObjects.py => interfaceDbChatbot.py} (95%) rename modules/interfaces/{interfaceDbRealEstateObjects.py => interfaceDbRealEstate.py} (93%) rename modules/interfaces/{interfaceDbTrusteeObjects.py => interfaceDbTrustee.py} (96%) rename modules/routes/{routeFeatureAutomation.py => routeFeatureWorkflow.py} (95%) create mode 100644 tool_db_export_migration.py create mode 100644 tool_db_import_migration.py diff --git a/app.py b/app.py index 7ed57ed9..b489194d 100644 --- a/app.py +++ b/app.py @@ -292,6 +292,10 @@ async def lifespan(app: FastAPI): # --- Init Managers --- await featuresLifecycle.start(eventUser) eventManager.start() + + # Register audit log cleanup scheduler + from modules.shared.auditLogger import registerAuditLogCleanupScheduler + registerAuditLogCleanupScheduler() yield @@ -444,7 +448,7 @@ app.include_router(sharepointRouter) from modules.routes.routeDataAutomation import router as automationRouter app.include_router(automationRouter) -from modules.routes.routeFeatureAutomation import router as adminAutomationEventsRouter +from modules.routes.routeFeatureWorkflow import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) from modules.routes.routeRbac import router as rbacRouter diff --git a/local/backup/migration_export_20260119_085558.json b/local/backup/migration_export_20260119_085558.json new file mode 100644 index 00000000..50cc9e5a --- /dev/null +++ b/local/backup/migration_export_20260119_085558.json @@ -0,0 +1,1221 @@ +{ + "meta": { + "exportedAt": "2026-01-19T07:55:59.185004Z", + "exportedFrom": "Development Instance Patrick", + "databaseName": "poweron_app", + "version": "1.0", + "tableCount": 6, + "excludedTables": [ + "_system" + ], + "includesMeta": false, + "totalRecords": 107 + }, + "tables": { + "AccessRule": [ + { + "id": "90990e75-ac45-4986-9c09-f299f198d2b3", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": null, + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "a9536849-a53b-458d-bab8-77a2a3ac2747", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": null, + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "n" + }, + { + "id": "54870935-d5a1-41b7-b088-30f70b4dc835", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": null, + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "a3743435-1015-4bd1-a256-4f91eda9f544", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": null, + "view": 1, + "read": "g", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "3e7b79b5-a68a-41b5-9e7f-f8159a310ed3", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Mandate", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "e7e8818c-04ed-473a-b1da-c7095f98f99e", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "61714126-2822-48cd-bbb5-175c141b2c0b", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0ae9dcb8-302a-44cb-b777-1f9d26af7190", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "2ab27800-df9d-4307-aee3-d99064fbab5d", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "279c6538-401a-4cd3-a36a-d1399f8bd2e4", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "4ab1abbe-abf1-4510-8ba1-4afbb37b1fd1", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "m", + "create": "n", + "update": "m", + "delete": "n" + }, + { + "id": "86c384a7-a2b5-4207-9e62-42cc50a2d20b", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "542af362-50c8-4e19-af21-5db2e0b8733e", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "8e356f1c-134d-4115-962e-0d0b7b71cf65", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "061b7a09-1003-4250-9524-64d648135467", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "89704bf9-d742-4149-a1e2-dcaaba9957c9", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "d296b5fb-48c3-4351-aa84-70ec1593369e", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "6ac892f8-69b6-4c2f-b18c-f5d58f8c95d9", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "defc2070-098f-44e2-9c59-264189730cc7", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "5927b71b-51f7-4671-a91b-f4f300af04f7", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "5a1907e7-6772-4b3f-9d34-6b847db3c14c", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "c4eac0f8-e9d2-4064-80bd-a4e9c9e68686", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "77a76ccf-91bb-4348-9caf-912b2bd7fe02", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "d059cf52-8c56-4273-80f9-e7b8b40ebb4b", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0ca0ed8b-9327-43ff-b7b5-dc5870fb09c2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "a72168c5-bf33-43ef-a708-fd40e8feb6ba", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "9c80b57b-d2ed-4309-9e87-62a98fe144d0", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "4e534940-33cc-43ac-b859-9360a2e60cf1", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "b0b3e37e-ea48-471e-95f6-b8e1e62ad09b", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "0f0f4c89-2aa5-4755-aeea-562bd020593d", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "0185dcc1-3560-4255-9840-8ab5db8042f8", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "c9c87948-21ce-47f4-8040-c02a0917aeb2", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "9e5f6dd5-c7a7-4a22-a1d6-82d4c68630a0", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "852f4f77-dac7-46de-8136-ebc46344101e", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "61710630-3c71-4adf-a997-b9e61d06fc00", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "158789bd-7f26-4d88-a27e-6ce11b3d94f9", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "c96b8b90-df7b-49ea-8ec9-0754c6179561", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "cdb610f9-21c6-4e8d-b9b6-d4a44f372ad6", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "6d3ff830-ae8f-4447-89a8-eaa8f8c4ecc7", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "7791781c-810a-43ac-a5ce-d84a3584e5f4", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "f49cd1cb-bdcd-4a9a-92fb-205078a3acba", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "44d5455e-a17b-4b6d-a426-21565fe1cef7", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "b2b58648-a555-4135-a038-84218a279437", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "073195c6-01bc-40f9-9896-f3c10ebce288", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "ba537ef0-239b-456d-b354-dec2c9d921ed", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "3d66cd04-f4e6-43f8-ab77-c31a87730096", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "bcc52c15-b41c-4a97-8e13-10035dd22202", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "844c7c02-dcc8-43a4-8eb5-f8ec198f50fd", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "15c1c76d-b8f7-40d2-8027-5ffdf6c96e28", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "b73379ef-3dbf-4118-b6f2-473f3c1c52e7", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "afc14c03-e953-4186-a923-4a12d3c6a312", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "c49ee8f3-f295-465d-be92-9bf46ee259e0", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "fcadb856-46f4-4ff0-85eb-67df3891c25d", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "4a4e52ad-4ee3-4044-9e14-e0d797356139", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "297ae474-ee2d-42f0-af09-eaa8b48d9684", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "a567d373-00be-4c7e-b25e-deb6b10b57a3", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "8f961779-1790-4cad-bebc-9b9bc436a29c", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "81be065a-9ec6-4713-9fca-f5087cb9c3ee", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "1fe2a920-0f9d-4c33-9288-4381af8c523e", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "e99b39eb-ed55-4d08-a5d5-c3cf4645c312", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "a54eed2a-e269-4884-ac2f-c09990f9b3d2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "2bd073ca-30c3-43e0-8ea1-d77241053e7c", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "557ba1d5-2be0-43ee-a842-d0e74b45df41", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "81b46bcc-6cc3-40e0-9d82-f3a5bcc23b53", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "f3a58145-8986-42a7-bce4-eab0a1fa0962", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "31d2a772-ed6b-447b-aa95-16a160d39601", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "8a373342-8b8b-4a64-afd4-9f3ae8071d28", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "b7a501e1-793f-47b6-b20c-1bfa61531cee", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0e67e101-5ce6-40fb-a783-104eebe29f92", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "23005b91-dc46-4bbe-a690-6c4568484858", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "aec6753b-cd7d-4ec8-8fa6-b9d007eb3657", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "31bf3c95-1012-4d61-91fc-36e4e3b832ec", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "17edb067-e62e-489a-842b-adbe4588aec0", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "981e57b7-d813-4d67-9cf9-73576cc2affb", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "3078980e-e8f2-44f0-b172-4a4c877b1b14", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "f25efabc-e861-4cba-ad9c-6c8886f99ae5", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "178c1dc4-7879-4a98-bbdd-54c1c55500c2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "bbf02493-5ba2-4c0d-83b7-2a1c20e41108", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "7183b536-9a8a-4c2e-b6f4-4f69cc8a50b4", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "309130ca-254d-45b0-8629-47fe6a391a34", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "240fdaa4-75df-47a4-b33e-bce2e09dc741", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "528ba11a-824d-477a-ae2f-aa453fdea2f2", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "e3efeb99-eeb2-4450-85a4-f4261449a9ea", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "ca4bf93c-311c-4d2b-8b6e-9bd879ff5cde", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "186bc21d-eb17-4d4a-ae4e-20725768befb", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "a", + "create": "n", + "update": "n", + "delete": "a" + }, + { + "id": "73d6eaa4-c1d1-4bfb-a2d6-a1772f9fd31c", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "a", + "create": "n", + "update": "n", + "delete": "a" + }, + { + "id": "2dce96a0-9a30-4d83-8544-29b60b7e8199", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "91b2e3ef-7472-41b0-81ef-7e891ce7d183", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "5bf4d7af-d8e9-4ca0-a9d1-c4cdd6a8b2a8", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "37999b49-c13c-44a1-a94e-41477d2e2e29", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "b7af378c-0da9-4554-ab9e-3e4d5130778b", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "d0c5bc55-d6e7-40c2-8bab-b7df13836712", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "96827d13-bb10-4341-9615-03b940e4eab1", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "c052f46c-07b7-447b-885d-15803f615b6a", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "91213ddd-9490-4f9f-928a-5f7c87bb6e05", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "58abcdc0-c8ba-435c-b01e-7b9c52581251", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + } + ], + "Mandate": [ + { + "id": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "name": "Root", + "enabled": 1 + } + ], + "Role": [ + { + "id": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "roleLabel": "sysadmin", + "description": { + "en": "System Administrator - Full access to all system resources", + "fr": "Administrateur système - Accès complet à toutes les ressources", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "roleLabel": "admin", + "description": { + "en": "Administrator - Manage users and resources within mandate scope", + "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "bc22885c-5354-463e-a3fe-480941e016df", + "roleLabel": "user", + "description": { + "en": "User - Standard user with access to own records", + "fr": "Utilisateur - Utilisateur standard avec accès à ses propres enregistrements", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "roleLabel": "viewer", + "description": { + "en": "Viewer - Read-only access to group records", + "fr": "Visualiseur - Accès en lecture seule aux enregistrements du groupe", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + } + ], + "UserInDB": [ + { + "id": "3876d530-29d8-451d-a2fc-92af5cb2b817", + "username": "admin", + "email": "admin@example.com", + "fullName": "Administrator", + "language": "en", + "enabled": 1, + "isSysAdmin": 1, + "roleLabels": [ + "sysadmin" + ], + "authenticationAuthority": "local", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$Sumds1bqvZfyPseYs/YeYw$eiRcnO7J+ebit2oV6Ndqaer2ZIgPErTC9q2riRpiiwA", + "resetToken": null, + "resetTokenExpires": null + }, + { + "id": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", + "username": "event", + "email": "event@example.com", + "fullName": "Event", + "language": "en", + "enabled": 1, + "isSysAdmin": 1, + "roleLabels": [ + "sysadmin" + ], + "authenticationAuthority": "local", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$BoDwnlNq7d3b2ztnrPWecw$ICkAaTjE/R39CJ7MryLmfmeEX5m4N/6S3HaDfOZuOBM", + "resetToken": null, + "resetTokenExpires": null + } + ], + "UserMandate": [ + { + "id": "70f9e733-7297-4afe-8784-9f7c730de3c4", + "userId": "3876d530-29d8-451d-a2fc-92af5cb2b817", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "enabled": 1 + }, + { + "id": "412ba93d-2abe-4916-8a80-bec4672c0baf", + "userId": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "enabled": 1 + } + ], + "UserMandateRole": [ + { + "id": "cff2dd70-4dd6-4028-a384-7b0e8578f9dc", + "userMandateId": "70f9e733-7297-4afe-8784-9f7c730de3c4", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" + }, + { + "id": "54232df1-b559-44bb-8a51-adbd82818d94", + "userMandateId": "412ba93d-2abe-4916-8a80-bec4672c0baf", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" + } + ] + }, + "summary": { + "AccessRule": { + "recordCount": 96 + }, + "Mandate": { + "recordCount": 1 + }, + "Role": { + "recordCount": 4 + }, + "UserInDB": { + "recordCount": 2 + }, + "UserMandate": { + "recordCount": 2 + }, + "UserMandateRole": { + "recordCount": 2 + } + } +} \ No newline at end of file diff --git a/modules/datamodels/datamodelAudit.py b/modules/datamodels/datamodelAudit.py new file mode 100644 index 00000000..76c9ecfb --- /dev/null +++ b/modules/datamodels/datamodelAudit.py @@ -0,0 +1,208 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Audit Log Data Model for database-based audit logging. + +This model stores security-relevant audit events for GDPR compliance and security monitoring. + +GDPR-Relevant Events: +- User access: login, logout, failed login attempts +- Data access: create, read, update, delete operations on personal data +- Security events: password changes, token refresh, session management +- Key access: encryption/decryption of sensitive data +- GDPR actions: data export, data portability, account deletion +- Mandate/permission changes: user added/removed from mandates, role changes +""" + +from typing import Optional +from pydantic import BaseModel, Field +from enum import Enum +import uuid + +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared.attributeUtils import registerModelLabels + + +class AuditCategory(str, Enum): + """Categories for audit log entries""" + ACCESS = "access" # Login/logout events + KEY = "key" # Encryption key access + DATA = "data" # Data CRUD operations + SECURITY = "security" # Security-related events + GDPR = "gdpr" # GDPR-specific actions + PERMISSION = "permission" # Permission/role changes + SYSTEM = "system" # System-level events + + +class AuditAction(str, Enum): + """Actions for audit log entries""" + # Access actions + LOGIN = "login" + LOGIN_FAILED = "login_failed" + LOGOUT = "logout" + TOKEN_REFRESH = "token_refresh" + TOKEN_REVOKE = "token_revoke" + SESSION_EXPIRED = "session_expired" + + # Key actions + KEY_ENCODE = "encode" + KEY_DECODE = "decode" + KEY_ACCESS = "key_access" + + # Data actions + DATA_CREATE = "create" + DATA_READ = "read" + DATA_UPDATE = "update" + DATA_DELETE = "delete" + DATA_EXPORT = "export" + + # Security actions + PASSWORD_CHANGE = "password_change" + PASSWORD_RESET = "password_reset" + MFA_ENABLED = "mfa_enabled" + MFA_DISABLED = "mfa_disabled" + + # GDPR actions + GDPR_DATA_EXPORT = "gdpr_data_export" + GDPR_DATA_PORTABILITY = "gdpr_data_portability" + GDPR_ACCOUNT_DELETION = "gdpr_account_deletion" + GDPR_CONSENT_UPDATE = "gdpr_consent_update" + + # Permission actions + USER_ADDED_TO_MANDATE = "user_added_to_mandate" + USER_REMOVED_FROM_MANDATE = "user_removed_from_mandate" + ROLE_ASSIGNED = "role_assigned" + ROLE_REVOKED = "role_revoked" + FEATURE_ACCESS_GRANTED = "feature_access_granted" + FEATURE_ACCESS_REVOKED = "feature_access_revoked" + + # System actions + SYSTEM_STARTUP = "system_startup" + SYSTEM_SHUTDOWN = "system_shutdown" + CONFIG_CHANGE = "config_change" + + +class AuditLogEntry(BaseModel): + """ + Audit log entry for database storage. + + Stores all security-relevant events for compliance and monitoring. + Entries are immutable once created (append-only audit trail). + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique identifier for the audit entry", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Timestamp + timestamp: float = Field( + default_factory=getUtcTimestamp, + description="UTC timestamp when the event occurred", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": True} + ) + + # Actor identification + userId: str = Field( + description="ID of the user who performed the action (or 'system' for system events)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + username: Optional[str] = Field( + default=None, + description="Username at the time of the event (for historical reference)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Context + mandateId: Optional[str] = Field( + default=None, + description="Mandate context (if applicable)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature instance context (if applicable)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Event classification + category: str = Field( + description="Event category (access, key, data, security, gdpr, permission, system)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + action: str = Field( + description="Specific action performed", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + # Event details + resourceType: Optional[str] = Field( + default=None, + description="Type of resource affected (e.g., 'User', 'ChatWorkflow', 'TrusteeContract')", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + resourceId: Optional[str] = Field( + default=None, + description="ID of the affected resource", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + details: Optional[str] = Field( + default=None, + description="Additional details about the event", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} + ) + + # Request metadata + ipAddress: Optional[str] = Field( + default=None, + description="IP address of the client", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + userAgent: Optional[str] = Field( + default=None, + description="User agent string from the request", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Outcome + success: bool = Field( + default=True, + description="Whether the action was successful", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": True} + ) + + errorMessage: Optional[str] = Field( + default=None, + description="Error message if the action failed", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} + ) + + +# Register labels for internationalization +registerModelLabels( + "AuditLogEntry", + {"en": "Audit Log Entry", "de": "Audit-Log-Eintrag", "fr": "Entrée du journal d'audit"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "timestamp": {"en": "Timestamp", "de": "Zeitstempel", "fr": "Horodatage"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "username": {"en": "Username", "de": "Benutzername", "fr": "Nom d'utilisateur"}, + "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "de": "Feature-Instanz-ID", "fr": "ID de l'instance"}, + "category": {"en": "Category", "de": "Kategorie", "fr": "Catégorie"}, + "action": {"en": "Action", "de": "Aktion", "fr": "Action"}, + "resourceType": {"en": "Resource Type", "de": "Ressourcentyp", "fr": "Type de ressource"}, + "resourceId": {"en": "Resource ID", "de": "Ressourcen-ID", "fr": "ID de ressource"}, + "details": {"en": "Details", "de": "Details", "fr": "Détails"}, + "ipAddress": {"en": "IP Address", "de": "IP-Adresse", "fr": "Adresse IP"}, + "userAgent": {"en": "User Agent", "de": "User-Agent", "fr": "Agent utilisateur"}, + "success": {"en": "Success", "de": "Erfolgreich", "fr": "Succès"}, + "errorMessage": {"en": "Error Message", "de": "Fehlermeldung", "fr": "Message d'erreur"}, + }, +) diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChatbot.py similarity index 95% rename from modules/datamodels/datamodelChat.py rename to modules/datamodels/datamodelChatbot.py index 7860658c..45c5c4eb 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChatbot.py @@ -14,6 +14,12 @@ class ChatStat(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this stat belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this stat belongs to" + ) workflowId: Optional[str] = Field( None, description="Foreign key to workflow (for workflow stats)" ) @@ -33,6 +39,8 @@ registerModelLabels( {"en": "Chat Statistics", "fr": "Statistiques de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, @@ -49,6 +57,12 @@ class ChatLog(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this log belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this log belongs to" + ) workflowId: str = Field(description="Foreign key to workflow") message: str = Field(description="Log message") type: str = Field(description="Log type (info, warning, error, etc.)") @@ -79,6 +93,8 @@ registerModelLabels( {"en": "Chat Log", "fr": "Journal de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "message": {"en": "Message", "fr": "Message"}, "type": {"en": "Type", "fr": "Type"}, @@ -94,6 +110,12 @@ class ChatDocument(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this document belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to" + ) messageId: str = Field(description="Foreign key to message") fileId: str = Field(description="Foreign key to file") fileName: str = Field(description="Name of the file") @@ -112,6 +134,8 @@ registerModelLabels( {"en": "Chat Document", "fr": "Document de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "messageId": {"en": "Message ID", "fr": "ID du message"}, "fileId": {"en": "File ID", "fr": "ID du fichier"}, "fileName": {"en": "File Name", "fr": "Nom du fichier"}, @@ -200,6 +224,12 @@ class ChatMessage(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this message belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this message belongs to" + ) workflowId: str = Field(description="Foreign key to workflow") parentMessageId: Optional[str] = Field( None, description="Parent message ID for threading" @@ -251,6 +281,8 @@ registerModelLabels( {"en": "Chat Message", "fr": "Message de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, "documents": {"en": "Documents", "fr": "Documents"}, @@ -296,6 +328,7 @@ registerModelLabels( class ChatWorkflow(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, @@ -370,6 +403,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "status": {"en": "Status", "fr": "Statut"}, "name": {"en": "Name", "fr": "Nom"}, "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, @@ -993,6 +1027,7 @@ registerModelLabels( class AutomationDefinition(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, @@ -1013,6 +1048,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "label": {"en": "Label", "fr": "Libellé"}, "schedule": {"en": "Schedule", "fr": "Planification"}, "template": {"en": "Template", "fr": "Modèle"}, diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index 5deafa42..07880f3d 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -13,6 +13,7 @@ import base64 class FileItem(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileName: str = Field(description="Name of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) mimeType: str = Field(description="MIME type of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileHash: str = Field(description="Hash of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) @@ -25,6 +26,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "fileName": {"en": "fileName", "fr": "Nom de fichier"}, "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, diff --git a/modules/datamodels/datamodelMessaging.py b/modules/datamodels/datamodelMessaging.py index 2ec09c40..1c2206b7 100644 --- a/modules/datamodels/datamodelMessaging.py +++ b/modules/datamodels/datamodelMessaging.py @@ -45,6 +45,10 @@ class MessagingSubscription(BaseModel): description="ID of the mandate this subscription belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + featureInstanceId: str = Field( + description="ID of the feature instance this subscription belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) description: Optional[str] = Field( default=None, description="Description of the subscription", @@ -92,6 +96,7 @@ registerModelLabels( "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "subscriptionLabel": {"en": "Subscription Label", "fr": "Label d'abonnement"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "description": {"en": "Description", "fr": "Description"}, "isSystemSubscription": {"en": "System Subscription", "fr": "Abonnement système"}, "enabled": {"en": "Enabled", "fr": "Activé"}, @@ -110,6 +115,14 @@ class MessagingSubscriptionRegistration(BaseModel): description="Unique ID of the registration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + mandateId: str = Field( + description="ID of the mandate this registration belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureInstanceId: str = Field( + description="ID of the feature instance this registration belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) subscriptionId: str = Field( description="ID of the subscription this registration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} @@ -161,6 +174,8 @@ registerModelLabels( {"en": "Messaging Registration", "fr": "Inscription à la messagerie"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "channel": {"en": "Channel", "fr": "Canal"}, @@ -179,6 +194,14 @@ class MessagingDelivery(BaseModel): description="Unique ID of the delivery", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + mandateId: str = Field( + description="ID of the mandate this delivery belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureInstanceId: str = Field( + description="ID of the feature instance this delivery belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) subscriptionId: str = Field( description="ID of the subscription this delivery belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} @@ -239,6 +262,8 @@ registerModelLabels( {"en": "Messaging Delivery", "fr": "Livraison de messagerie"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "channel": {"en": "Channel", "fr": "Canal"}, diff --git a/modules/datamodels/datamodelNeutralizer.py b/modules/datamodels/datamodelNeutralizer.py index 9d92bd60..e7b46c4d 100644 --- a/modules/datamodels/datamodelNeutralizer.py +++ b/modules/datamodels/datamodelNeutralizer.py @@ -11,6 +11,7 @@ from modules.shared.attributeUtils import registerModelLabels class DataNeutraliserConfig(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) userId: str = Field(description="ID of the user who created this configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) enabled: bool = Field(default=True, description="Whether data neutralization is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) namesToParse: str = Field(default="", description="Multiline list of names to parse for neutralization", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False}) @@ -22,6 +23,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "enabled": {"en": "Enabled", "fr": "Activé"}, "namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"}, @@ -33,6 +35,7 @@ registerModelLabels( class DataNeutralizerAttributes(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the attribute mapping (used as UID in neutralized files)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) userId: str = Field(description="ID of the user who created this attribute", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) originalText: str = Field(description="Original text that was neutralized", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) fileId: Optional[str] = Field(default=None, description="ID of the file this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) @@ -43,6 +46,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "originalText": {"en": "Original Text", "fr": "Texte original"}, "fileId": {"en": "File ID", "fr": "ID de fichier"}, diff --git a/modules/datamodels/datamodelRealEstate.py b/modules/datamodels/datamodelRealEstate.py index fa9e717e..31efbc07 100644 --- a/modules/datamodels/datamodelRealEstate.py +++ b/modules/datamodels/datamodelRealEstate.py @@ -123,6 +123,12 @@ class Dokument(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Document label", frontend_type="text", @@ -207,6 +213,12 @@ class Land(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Country name (e.g. 'Schweiz')", frontend_type="text", @@ -251,6 +263,12 @@ class Kanton(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Canton name (e.g. 'Zürich')", frontend_type="text", @@ -302,6 +320,12 @@ class Gemeinde(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Municipality name (e.g. 'Zürich')", frontend_type="text", @@ -359,6 +383,12 @@ class Parzelle(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) # Grunddaten label: str = Field( @@ -579,6 +609,12 @@ class Projekt(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Project designation", frontend_type="text", @@ -643,6 +679,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "statusProzess": {"en": "Process Status", "fr": "Statut du processus", "de": "Prozessstatus"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) @@ -653,6 +690,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID", "de": "ID"}, "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) @@ -662,6 +700,8 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID", "de": "ID"}, "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) diff --git a/modules/datamodels/datamodelTrustee.py b/modules/datamodels/datamodelTrustee.py index 21a3d3cc..e2d9b261 100644 --- a/modules/datamodels/datamodelTrustee.py +++ b/modules/datamodels/datamodelTrustee.py @@ -44,6 +44,15 @@ class TrusteeOrganisation(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector: # _createdAt, _modifiedAt, _createdBy, _modifiedBy @@ -56,6 +65,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "enabled": {"en": "Enabled", "fr": "Activé", "de": "Aktiviert"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -87,6 +97,15 @@ class TrusteeRole(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -97,6 +116,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID", "de": "ID"}, "desc": {"en": "Description", "fr": "Description", "de": "Beschreibung"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -159,6 +179,15 @@ class TrusteeAccess(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -172,6 +201,7 @@ registerModelLabels( "userId": {"en": "User", "fr": "Utilisateur", "de": "Benutzer"}, "contractId": {"en": "Contract (optional)", "fr": "Contrat (optionnel)", "de": "Vertrag (optional)"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -222,6 +252,15 @@ class TrusteeContract(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -234,6 +273,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "enabled": {"en": "Enabled", "fr": "Activé", "de": "Aktiviert"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -309,6 +349,15 @@ class TrusteeDocument(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -323,6 +372,7 @@ registerModelLabels( "documentName": {"en": "Document Name", "fr": "Nom du document", "de": "Dokumentname"}, "documentMimeType": {"en": "MIME Type", "fr": "Type MIME", "de": "MIME-Typ"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -477,6 +527,15 @@ class TrusteePosition(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -499,6 +558,7 @@ registerModelLabels( "vatPercentage": {"en": "VAT Percentage", "fr": "Pourcentage TVA", "de": "MwSt-Prozentsatz"}, "vatAmount": {"en": "VAT Amount", "fr": "Montant TVA", "de": "MwSt-Betrag"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -562,6 +622,15 @@ class TrusteePositionDocument(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -575,5 +644,6 @@ registerModelLabels( "documentId": {"en": "Document", "fr": "Document", "de": "Dokument"}, "positionId": {"en": "Position", "fr": "Position", "de": "Position"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py index bb1ed9ca..86f4bb1d 100644 --- a/modules/datamodels/datamodelVoice.py +++ b/modules/datamodels/datamodelVoice.py @@ -12,6 +12,7 @@ class VoiceSettings(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) mandateId: str = Field(description="ID of the mandate these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) @@ -28,6 +29,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py index b884382c..19117fce 100644 --- a/modules/datamodels/datamodelWorkflow.py +++ b/modules/datamodels/datamodelWorkflow.py @@ -14,7 +14,7 @@ from modules.datamodels.datamodelDocref import DocumentReferenceList # Forward references for circular imports (use string annotations) if TYPE_CHECKING: - from modules.datamodels.datamodelChat import ChatDocument, ActionResult + from modules.datamodels.datamodelChatbot import ChatDocument, ActionResult from modules.datamodels.datamodelExtraction import ExtractionOptions diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index 8bac1fd5..1ca90d51 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 43503339..a5222966 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -13,7 +13,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -335,7 +335,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChat import ChatLog + from modules.datamodels.datamodelChatbot import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 37c4a3cd..0d386aac 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -23,7 +23,7 @@ from modules.datamodels.datamodelRealEstate import ( Land, ) from modules.services import getInterface as getServices -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector logger = logging.getLogger(__name__) diff --git a/modules/features/workflow/mainWorkflow.py b/modules/features/workflow/mainWorkflow.py index 70a2e9aa..ab92510c 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/features/workflow/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/features/workflow/subAutomationUtils.py b/modules/features/workflow/subAutomationUtils.py index 60993b62..906c9caa 100644 --- a/modules/features/workflow/subAutomationUtils.py +++ b/modules/features/workflow/subAutomationUtils.py @@ -3,7 +3,7 @@ """ Utility functions for automation feature. -Moved from interfaces/interfaceDbChatObjects.py. +Moved from interfaces/interfaceDbChatbot.py. """ import json diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatbot.py similarity index 95% rename from modules/interfaces/interfaceDbChatObjects.py rename to modules/interfaces/interfaceDbChatbot.py index 10b202da..9ded2dd8 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( ChatDocument, ChatStat, ChatLog, @@ -178,18 +178,20 @@ class ChatObjects: Uses the JSON connector for data access with added language support. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Chat Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ # Initialize variables self.currentUser = currentUser # Store User object directly self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # RBAC interface # Initialize services @@ -200,7 +202,7 @@ class ChatObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) # ===== Generic Utility Methods ===== @@ -263,17 +265,19 @@ class ChatObjects: def _initializeServices(self): pass - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser # Store User object directly self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -603,10 +607,12 @@ class ChatObjects: If pagination is None: List[Dict[str, Any]] If pagination is provided: PaginatedResult with items and metadata """ - # Use RBAC filtering + # Use RBAC filtering with featureInstanceId for instance-level isolation filteredWorkflows = getRecordsetWithRBAC(self.db, ChatWorkflow, - self.currentUser + self.currentUser, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # If no pagination requested, return all items (no sorting - frontend handles it) @@ -638,11 +644,13 @@ class ChatObjects: def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: """Returns a workflow by ID if user has access.""" - # Use RBAC filtering + # Use RBAC filtering with featureInstanceId for instance-level isolation workflows = getRecordsetWithRBAC(self.db, ChatWorkflow, self.currentUser, - recordFilter={"id": workflowId} + recordFilter={"id": workflowId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if not workflows: @@ -689,6 +697,12 @@ class ChatObjects: if "lastActivity" not in workflowData: workflowData["lastActivity"] = currentTime + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in workflowData or not workflowData["mandateId"]: + workflowData["mandateId"] = self.mandateId + if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: + workflowData["featureInstanceId"] = self.featureInstanceId + # Use generic field separation based on ChatWorkflow model simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) @@ -1009,6 +1023,12 @@ class ChatObjects: if "actionNumber" not in messageData: messageData["actionNumber"] = workflow.currentAction + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in messageData or not messageData["mandateId"]: + messageData["mandateId"] = self.mandateId + if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: + messageData["featureInstanceId"] = self.featureInstanceId + # Use generic field separation based on ChatMessage model simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) @@ -1303,6 +1323,12 @@ class ChatObjects: def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: """Creates a document for a message in normalized table.""" try: + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in documentData or not documentData["mandateId"]: + documentData["mandateId"] = self.mandateId + if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: + documentData["featureInstanceId"] = self.featureInstanceId + # Validate and normalize document data to dict document = ChatDocument(**documentData) logger.debug(f"Creating document in database: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") @@ -1422,6 +1448,12 @@ class ChatObjects: if "timestamp" not in logData: logData["timestamp"] = getUtcTimestamp() + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in logData or not logData["mandateId"]: + logData["mandateId"] = self.mandateId + if "featureInstanceId" not in logData or not logData["featureInstanceId"]: + logData["featureInstanceId"] = self.featureInstanceId + # Add status information if not present if "status" not in logData and "type" in logData: if logData["type"] == "error": @@ -1508,6 +1540,12 @@ class ChatObjects: if "workflowId" not in statData: raise ValueError("workflowId is required in statData") + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in statData or not statData["mandateId"]: + statData["mandateId"] = self.mandateId + if "featureInstanceId" not in statData or not statData["featureInstanceId"]: + statData["featureInstanceId"] = self.featureInstanceId + # Validate the stat data against ChatStat model stat = ChatStat(**statData) @@ -1768,9 +1806,11 @@ class ChatObjects: if "id" not in automationData or not automationData["id"]: automationData["id"] = str(uuid.uuid4()) - # Ensure mandateId is set + # Ensure mandateId and featureInstanceId are set for proper data isolation if "mandateId" not in automationData: automationData["mandateId"] = self.mandateId + if "featureInstanceId" not in automationData: + automationData["featureInstanceId"] = self.featureInstanceId # Ensure database connector has correct userId context # The connector should have been initialized with userId, but ensure it's updated @@ -1894,7 +1934,7 @@ class ChatObjects: logger.error(f"Error notifying automation change: {str(e)}") -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ChatObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': """ Returns a ChatObjects instance for the current user. Handles initialization of database and records. @@ -1902,20 +1942,22 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ if not currentUser: raise ValueError("Invalid user context: user is required") effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - # Create context key - contextKey = f"{effectiveMandateId}_{currentUser.id}" + # Create context key including featureInstanceId for proper isolation + contextKey = f"{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" # Create new instance if not exists if contextKey not in _chatInterfaces: - _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId) + _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _chatInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbComponentObjects.py index 07cf235c..72d65839 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbComponentObjects.py @@ -76,12 +76,13 @@ class ComponentObjects: # Initialize standard records if needed self._initRecords() - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ if not currentUser: logger.info("Initializing interface without user context") @@ -91,6 +92,7 @@ class ComponentObjects: self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -986,12 +988,14 @@ class ComponentObjects: fileSize = len(content) fileHash = hashlib.sha256(content).hexdigest() - # Use mandateId from context + # Use mandateId and featureInstanceId from context for proper data isolation mandateId = self.mandateId + featureInstanceId = self.featureInstanceId # Create FileItem instance fileItem = FileItem( mandateId=mandateId, + featureInstanceId=featureInstanceId, fileName=uniqueName, mimeType=mimeType, fileSize=fileSize, @@ -1327,9 +1331,11 @@ class ComponentObjects: if "userId" not in settingsData: settingsData["userId"] = self.userId - # Ensure mandateId is set from context + # Ensure mandateId and featureInstanceId are set from context if "mandateId" not in settingsData: settingsData["mandateId"] = self.mandateId + if "featureInstanceId" not in settingsData: + settingsData["featureInstanceId"] = self.featureInstanceId # Check if settings already exist for this user existingSettings = self.getVoiceSettings(settingsData["userId"]) @@ -1501,9 +1507,11 @@ class ComponentObjects: if not all(c.isalpha() or c == "_" for c in subscriptionId): raise ValueError("subscriptionId must contain only letters and underscores") - # Set mandateId from context + # Set mandateId and featureInstanceId from context for proper data isolation if "mandateId" not in subscriptionData: subscriptionData["mandateId"] = self.mandateId + if "featureInstanceId" not in subscriptionData: + subscriptionData["featureInstanceId"] = self.featureInstanceId createdRecord = self.db.recordCreate(MessagingSubscription, subscriptionData) if not createdRecord or not createdRecord.get("id"): @@ -1605,6 +1613,12 @@ class ComponentObjects: if "userId" not in registrationData: registrationData["userId"] = self.userId + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in registrationData: + registrationData["mandateId"] = self.mandateId + if "featureInstanceId" not in registrationData: + registrationData["featureInstanceId"] = self.featureInstanceId + createdRecord = self.db.recordCreate(MessagingSubscriptionRegistration, registrationData) if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create registration record") @@ -1679,6 +1693,13 @@ class ComponentObjects: def createDelivery(self, delivery: MessagingDelivery) -> Dict[str, Any]: """Creates a new delivery record.""" deliveryData = delivery.model_dump() if isinstance(delivery, MessagingDelivery) else delivery + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in deliveryData or not deliveryData["mandateId"]: + deliveryData["mandateId"] = self.mandateId + if "featureInstanceId" not in deliveryData or not deliveryData["featureInstanceId"]: + deliveryData["featureInstanceId"] = self.featureInstanceId + createdRecord = self.db.recordCreate(MessagingDelivery, deliveryData) if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create delivery record") @@ -1748,7 +1769,7 @@ class ComponentObjects: return MessagingDelivery(**filteredDeliveries[0]) if filteredDeliveries else None -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ComponentObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ComponentObjects': """ Returns a ComponentObjects instance. If currentUser is provided, initializes with user context. @@ -1757,8 +1778,10 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None # Create new instance if not exists if "default" not in _instancesManagement: @@ -1767,7 +1790,7 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = interface = _instancesManagement["default"] if currentUser: - interface.setUserContext(currentUser, mandateId=effectiveMandateId) + interface.setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: logger.info("Returning interface without user context") diff --git a/modules/interfaces/interfaceDbRealEstateObjects.py b/modules/interfaces/interfaceDbRealEstate.py similarity index 93% rename from modules/interfaces/interfaceDbRealEstateObjects.py rename to modules/interfaces/interfaceDbRealEstate.py index d475b8db..179ec6dd 100644 --- a/modules/interfaces/interfaceDbRealEstateObjects.py +++ b/modules/interfaces/interfaceDbRealEstate.py @@ -39,17 +39,19 @@ class RealEstateObjects: Handles CRUD operations on Real Estate entities. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Real Estate Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # RBAC interface # Initialize database @@ -57,7 +59,7 @@ class RealEstateObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) def _initializeDatabase(self): """Initialize PostgreSQL database connection.""" @@ -107,17 +109,19 @@ class RealEstateObjects: logger.warning(f"Error ensuring supporting tables exist: {e}") # Don't raise - tables will be created on-demand anyway - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -145,9 +149,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Projekt, "create"): raise PermissionError(f"User {self.userId} cannot create projects") - # Ensure mandateId is set + # Ensure mandateId and featureInstanceId are set for proper data isolation if not projekt.mandateId: projekt.mandateId = self.mandateId + if not projekt.featureInstanceId: + projekt.featureInstanceId = self.featureInstanceId # Save to database - use mode='json' to ensure nested Pydantic models are serialized self.db.recordCreate(Projekt, projekt.model_dump(mode='json')) @@ -231,8 +237,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Parzelle, "create"): raise PermissionError(f"User {self.userId} cannot create plots") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not parzelle.mandateId: parzelle.mandateId = self.mandateId + if not parzelle.featureInstanceId: + parzelle.featureInstanceId = self.featureInstanceId # Use mode='json' to ensure nested Pydantic models (like GeoPolylinie) are serialized self.db.recordCreate(Parzelle, parzelle.model_dump(mode='json')) @@ -438,8 +447,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Dokument, "create"): raise PermissionError(f"User {self.userId} cannot create documents") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not dokument.mandateId: dokument.mandateId = self.mandateId + if not dokument.featureInstanceId: + dokument.featureInstanceId = self.featureInstanceId self.db.recordCreate(Dokument, dokument.model_dump()) @@ -504,8 +516,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Gemeinde, "create"): raise PermissionError(f"User {self.userId} cannot create municipalities") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not gemeinde.mandateId: gemeinde.mandateId = self.mandateId + if not gemeinde.featureInstanceId: + gemeinde.featureInstanceId = self.featureInstanceId self.db.recordCreate(Gemeinde, gemeinde.model_dump()) @@ -570,8 +585,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Kanton, "create"): raise PermissionError(f"User {self.userId} cannot create cantons") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not kanton.mandateId: kanton.mandateId = self.mandateId + if not kanton.featureInstanceId: + kanton.featureInstanceId = self.featureInstanceId self.db.recordCreate(Kanton, kanton.model_dump()) @@ -636,8 +654,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Land, "create"): raise PermissionError(f"User {self.userId} cannot create countries") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not land.mandateId: land.mandateId = self.mandateId + if not land.featureInstanceId: + land.featureInstanceId = self.featureInstanceId self.db.recordCreate(Land, land.model_dump()) @@ -792,7 +813,7 @@ class RealEstateObjects: raise -def getInterface(currentUser: User, mandateId: Optional[str] = None) -> RealEstateObjects: +def getInterface(currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> RealEstateObjects: """ Factory function to get or create a Real Estate interface instance for a user. Uses singleton pattern per user. @@ -800,16 +821,19 @@ def getInterface(currentUser: User, mandateId: Optional[str] = None) -> RealEsta Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - userKey = f"{currentUser.id}_{effectiveMandateId}" + # Include featureInstanceId in key for proper isolation + userKey = f"{currentUser.id}_{effectiveMandateId}_{effectiveFeatureInstanceId}" if userKey not in _realEstateInterfaces: - _realEstateInterfaces[userKey] = RealEstateObjects(currentUser, mandateId=effectiveMandateId) + _realEstateInterfaces[userKey] = RealEstateObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _realEstateInterfaces[userKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _realEstateInterfaces[userKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _realEstateInterfaces[userKey] diff --git a/modules/interfaces/interfaceDbTrusteeObjects.py b/modules/interfaces/interfaceDbTrustee.py similarity index 96% rename from modules/interfaces/interfaceDbTrusteeObjects.py rename to modules/interfaces/interfaceDbTrustee.py index edb085fa..56ce19b3 100644 --- a/modules/interfaces/interfaceDbTrusteeObjects.py +++ b/modules/interfaces/interfaceDbTrustee.py @@ -33,12 +33,13 @@ logger = logging.getLogger(__name__) _trusteeInterfaces = {} -def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = None) -> "TrusteeObjects": +def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = None, featureInstanceId: Optional[str] = None) -> "TrusteeObjects": """Get or create a TrusteeObjects instance for the given user context. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ global _trusteeInterfaces @@ -46,14 +47,16 @@ def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = raise ValueError("Valid user context required") effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - cacheKey = f"{currentUser.id}_{effectiveMandateId}" + # Include featureInstanceId in cache key for proper isolation + cacheKey = f"{currentUser.id}_{effectiveMandateId}_{effectiveFeatureInstanceId}" if cacheKey not in _trusteeInterfaces: - _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser, mandateId=effectiveMandateId) + _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _trusteeInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _trusteeInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _trusteeInterfaces[cacheKey] @@ -64,17 +67,19 @@ class TrusteeObjects: Manages trustee organisations, roles, access, contracts, documents, and positions. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Trustee Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # Initialize database @@ -82,14 +87,15 @@ class TrusteeObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ if not currentUser: logger.info("Initializing interface without user context") @@ -99,6 +105,7 @@ class TrusteeObjects: self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -204,8 +211,10 @@ class TrusteeObjects: logger.warning(f"User {self.userId} lacks permission to create organisation") return None - # Set mandateId from current user + # Set mandateId and featureInstanceId from context for proper data isolation data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId # Validate ID format (alphanumeric, hyphens, underscores, 3-50 chars) orgId = data.get("id", "") @@ -307,6 +316,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId roleId = data.get("id", "") if not roleId: @@ -414,6 +425,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid accessId = data.get("id") or str(uuid.uuid4()) @@ -603,6 +616,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid contractId = data.get("id") or str(uuid.uuid4()) @@ -729,6 +744,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid documentId = data.get("id") or str(uuid.uuid4()) @@ -879,6 +896,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId # Calculate VAT amount if not provided if "vatAmount" not in data or data.get("vatAmount") == 0: @@ -1028,6 +1047,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid linkId = data.get("id") or str(uuid.uuid4()) diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 3b5515df..071f8673 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -13,9 +13,9 @@ import logging import json # Import interfaces and models -from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface +from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow +from modules.datamodels.datamodelChatbot import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.features.workflow import executeAutomation diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index a118869a..bf3fbc73 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -472,12 +472,15 @@ async def addUserToMandate( roleIds=data.roleIds ) - # 8. Audit - audit_logger.logSecurityEvent( + # 8. Audit - Log permission change with IP address + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, action="user_added_to_mandate", - details=f"targetUser={data.targetUserId}, roles={data.roleIds}" + targetUserId=data.targetUserId, + details=f"Roles assigned: {data.roleIds}", + resourceType="UserMandate", + resourceId=str(userMandate.id) ) logger.info( @@ -557,12 +560,14 @@ async def removeUserFromMandate( # Delete UserMandate (CASCADE will delete UserMandateRole entries) rootInterface.deleteUserMandate(targetUserId, mandateId) - # Audit - audit_logger.logSecurityEvent( + # Audit - Log permission change + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, action="user_removed_from_mandate", - details=f"targetUser={targetUserId}" + targetUserId=targetUserId, + details="User removed from mandate", + resourceType="UserMandate" ) logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {mandateId}") @@ -657,12 +662,15 @@ async def updateUserRolesInMandate( for roleId in roleIds: rootInterface.addRoleToUserMandate(str(membership.id), roleId) - # Audit - audit_logger.logSecurityEvent( + # Audit - Log role assignment change + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, - action="user_roles_updated_in_mandate", - details=f"targetUser={targetUserId}, newRoles={roleIds}" + action="role_assigned", + targetUserId=targetUserId, + details=f"New roles: {roleIds}", + resourceType="UserMandateRole", + resourceId=str(membership.id) ) logger.info( diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index ae6ab70a..226f3606 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -360,7 +360,9 @@ async def reset_user_password( userId=str(context.user.id), mandateId=str(context.mandateId) if context.mandateId else "system", action="password_reset", - details=f"Reset password for user {userId}" + details=f"Reset password for user {userId}", + ipAddress=request.client.host if request.client else None, + success=True ) except Exception: pass @@ -439,7 +441,9 @@ async def change_password( userId=str(context.user.id), mandateId=str(context.mandateId) if context.mandateId else "system", action="password_change", - details="User changed their own password" + details="User changed their own password", + ipAddress=request.client.host if request.client else None, + success=True ) except Exception: pass diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index f2955b61..d6f53d84 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -13,10 +13,10 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions from modules.features.workflow import chatStart, chatStop @@ -32,7 +32,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Workflow start endpoint @router.post("/start", response_model=ChatWorkflow) diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index 0505a752..20b90876 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -18,11 +18,11 @@ from modules.shared.timeUtils import parseTimestamp from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature @@ -43,7 +43,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Chatbot streaming endpoint (SSE) @router.post("/start/stream") diff --git a/modules/routes/routeFeatureRealEstate.py b/modules/routes/routeFeatureRealEstate.py index fe7544de..7e130c1b 100644 --- a/modules/routes/routeFeatureRealEstate.py +++ b/modules/routes/routeFeatureRealEstate.py @@ -26,7 +26,7 @@ from modules.datamodels.datamodelRealEstate import ( ) # Import interfaces -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands from modules.features.realEstate.mainRealEstate import ( diff --git a/modules/routes/routeFeatureTrustee.py b/modules/routes/routeFeatureTrustee.py index 69fd5918..ad842db8 100644 --- a/modules/routes/routeFeatureTrustee.py +++ b/modules/routes/routeFeatureTrustee.py @@ -3,6 +3,10 @@ """ Routes for Trustee feature data management. Implements CRUD operations for organisations, roles, access, contracts, documents, and positions. + +URL Structure: /api/trustee/{instanceId}/{entity} +- instanceId is the FeatureInstance ID (required for all operations) +- This ensures proper multi-tenant isolation at the URL level """ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Query, Response @@ -14,7 +18,9 @@ import json import io from modules.auth import limiter, getRequestContext, RequestContext -from modules.interfaces.interfaceDbTrusteeObjects import getInterface +from modules.interfaces.interfaceDbTrustee import getInterface +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface from modules.datamodels.datamodelTrustee import ( TrusteeOrganisation, TrusteeRole, @@ -59,22 +65,72 @@ def _parsePagination(pagination: Optional[str]) -> Optional[PaginationParams]: return None +async def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: + """ + Validate that the user has access to the feature instance. + Returns the mandateId for the instance. + + Args: + instanceId: The FeatureInstance ID from URL + context: The request context with user info + + Returns: + mandateId of the instance + + Raises: + HTTPException 404 if instance not found + HTTPException 403 if user doesn't have access + """ + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=404, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify it's a trustee instance + if instance.featureCode != "trustee": + raise HTTPException( + status_code=400, + detail=f"Instance '{instanceId}' is not a trustee instance" + ) + + # Verify user has access to this instance + if not context.isSysAdmin: + # Check if user has FeatureAccess for this instance + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + hasAccess = any( + str(fa.featureInstanceId) == instanceId and fa.enabled + for fa in featureAccesses + ) + if not hasAccess: + raise HTTPException( + status_code=403, + detail=f"Access denied to feature instance '{instanceId}'" + ) + + return str(instance.mandateId) + + # ===== Organisation Routes ===== -@router.get("/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) +@router.get("/{instanceId}/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) @limiter.limit("30/minute") async def getOrganisations( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeOrganisation]: - """Get all organisations with optional pagination.""" - logger = logging.getLogger(__name__) - logger.debug(f"getOrganisations called for user {context.user.id}, mandateId: {context.mandateId}") + """Get all organisations for a feature instance with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllOrganisations(paginationParams) - logger.debug(f"getOrganisations returned {len(result.items)} items") if paginationParams: return PaginatedResponse( @@ -91,46 +147,55 @@ async def getOrganisations( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/organisations/{orgId}", response_model=TrusteeOrganisation) +@router.get("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("30/minute") async def getOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Get a single organisation by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) org = interface.getOrganisation(orgId) if not org: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") return org -@router.post("/organisations", response_model=TrusteeOrganisation, status_code=201) +@router.post("/{instanceId}/organisations", response_model=TrusteeOrganisation, status_code=201) @limiter.limit("10/minute") async def createOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeOrganisation = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Create a new organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createOrganisation(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create organisation") return result -@router.put("/organisations/{orgId}", response_model=TrusteeOrganisation) +@router.put("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("10/minute") async def updateOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), data: TrusteeOrganisation = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Update an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -141,15 +206,18 @@ async def updateOrganisation( return result -@router.delete("/organisations/{orgId}") +@router.delete("/{instanceId}/organisations/{orgId}") @limiter.limit("10/minute") async def deleteOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -162,16 +230,19 @@ async def deleteOrganisation( # ===== Role Routes ===== -@router.get("/roles", response_model=PaginatedResponse[TrusteeRole]) +@router.get("/{instanceId}/roles", response_model=PaginatedResponse[TrusteeRole]) @limiter.limit("30/minute") async def getRoles( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeRole]: """Get all roles with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllRoles(paginationParams) if paginationParams: @@ -189,46 +260,55 @@ async def getRoles( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/roles/{roleId}", response_model=TrusteeRole) +@router.get("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("30/minute") async def getRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Get a single role by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) role = interface.getRole(roleId) if not role: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") return role -@router.post("/roles", response_model=TrusteeRole, status_code=201) +@router.post("/{instanceId}/roles", response_model=TrusteeRole, status_code=201) @limiter.limit("10/minute") async def createRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeRole = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Create a new role (sysadmin only).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createRole(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create role") return result -@router.put("/roles/{roleId}", response_model=TrusteeRole) +@router.put("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("10/minute") async def updateRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), data: TrusteeRole = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Update a role (sysadmin only).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -239,15 +319,18 @@ async def updateRole( return result -@router.delete("/roles/{roleId}") +@router.delete("/{instanceId}/roles/{roleId}") @limiter.limit("10/minute") async def deleteRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a role (sysadmin only, fails if in use).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -260,16 +343,19 @@ async def deleteRole( # ===== Access Routes ===== -@router.get("/access", response_model=PaginatedResponse[TrusteeAccess]) +@router.get("/{instanceId}/access", response_model=PaginatedResponse[TrusteeAccess]) @limiter.limit("30/minute") async def getAllAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeAccess]: """Get all access records with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllAccess(paginationParams) if paginationParams: @@ -287,70 +373,85 @@ async def getAllAccess( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/access/{accessId}", response_model=TrusteeAccess) +@router.get("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("30/minute") async def getAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Get a single access record by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) access = interface.getAccess(accessId) if not access: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") return access -@router.get("/access/organisation/{orgId}", response_model=List[TrusteeAccess]) +@router.get("/{instanceId}/access/organisation/{orgId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") async def getAccessByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getAccessByOrganisation(orgId) -@router.get("/access/user/{userId}", response_model=List[TrusteeAccess]) +@router.get("/{instanceId}/access/user/{userId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") async def getAccessByUser( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), userId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for a user.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getAccessByUser(userId) -@router.post("/access", response_model=TrusteeAccess, status_code=201) +@router.post("/{instanceId}/access", response_model=TrusteeAccess, status_code=201) @limiter.limit("10/minute") async def createAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeAccess = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Create a new access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createAccess(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create access") return result -@router.put("/access/{accessId}", response_model=TrusteeAccess) +@router.put("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("10/minute") async def updateAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), data: TrusteeAccess = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Update an access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -361,15 +462,18 @@ async def updateAccess( return result -@router.delete("/access/{accessId}") +@router.delete("/{instanceId}/access/{accessId}") @limiter.limit("10/minute") async def deleteAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -382,16 +486,19 @@ async def deleteAccess( # ===== Contract Routes ===== -@router.get("/contracts", response_model=PaginatedResponse[TrusteeContract]) +@router.get("/{instanceId}/contracts", response_model=PaginatedResponse[TrusteeContract]) @limiter.limit("30/minute") async def getContracts( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeContract]: """Get all contracts with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllContracts(paginationParams) if paginationParams: @@ -409,58 +516,70 @@ async def getContracts( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/contracts/{contractId}", response_model=TrusteeContract) +@router.get("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("30/minute") async def getContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Get a single contract by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) contract = interface.getContract(contractId) if not contract: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") return contract -@router.get("/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) +@router.get("/{instanceId}/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) @limiter.limit("30/minute") async def getContractsByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeContract]: """Get all contracts for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getContractsByOrganisation(orgId) -@router.post("/contracts", response_model=TrusteeContract, status_code=201) +@router.post("/{instanceId}/contracts", response_model=TrusteeContract, status_code=201) @limiter.limit("10/minute") async def createContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeContract = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Create a new contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createContract(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create contract") return result -@router.put("/contracts/{contractId}", response_model=TrusteeContract) +@router.put("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("10/minute") async def updateContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), data: TrusteeContract = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Update a contract (organisationId is immutable).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -471,15 +590,18 @@ async def updateContract( return result -@router.delete("/contracts/{contractId}") +@router.delete("/{instanceId}/contracts/{contractId}") @limiter.limit("10/minute") async def deleteContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -492,16 +614,19 @@ async def deleteContract( # ===== Document Routes ===== -@router.get("/documents", response_model=PaginatedResponse[TrusteeDocument]) +@router.get("/{instanceId}/documents", response_model=PaginatedResponse[TrusteeDocument]) @limiter.limit("30/minute") async def getDocuments( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeDocument]: """Get all documents (metadata only) with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllDocuments(paginationParams) if paginationParams: @@ -519,30 +644,36 @@ async def getDocuments( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/documents/{documentId}", response_model=TrusteeDocument) +@router.get("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("30/minute") async def getDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Get document metadata by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") return doc -@router.get("/documents/{documentId}/data") +@router.get("/{instanceId}/documents/{documentId}/data") @limiter.limit("10/minute") async def getDocumentData( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ): """Download document binary data.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -558,43 +689,52 @@ async def getDocumentData( ) -@router.get("/documents/contract/{contractId}", response_model=List[TrusteeDocument]) +@router.get("/{instanceId}/documents/contract/{contractId}", response_model=List[TrusteeDocument]) @limiter.limit("30/minute") async def getDocumentsByContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeDocument]: """Get all documents for a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getDocumentsByContract(contractId) -@router.post("/documents", response_model=TrusteeDocument, status_code=201) +@router.post("/{instanceId}/documents", response_model=TrusteeDocument, status_code=201) @limiter.limit("10/minute") async def createDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Create a new document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create document") return result -@router.put("/documents/{documentId}", response_model=TrusteeDocument) +@router.put("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("10/minute") async def updateDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), data: TrusteeDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Update document metadata.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -605,15 +745,18 @@ async def updateDocument( return result -@router.delete("/documents/{documentId}") +@router.delete("/{instanceId}/documents/{documentId}") @limiter.limit("10/minute") async def deleteDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -626,16 +769,19 @@ async def deleteDocument( # ===== Position Routes ===== -@router.get("/positions", response_model=PaginatedResponse[TrusteePosition]) +@router.get("/{instanceId}/positions", response_model=PaginatedResponse[TrusteePosition]) @limiter.limit("30/minute") async def getPositions( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePosition]: """Get all positions with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(paginationParams) if paginationParams: @@ -653,70 +799,85 @@ async def getPositions( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/positions/{positionId}", response_model=TrusteePosition) +@router.get("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("30/minute") async def getPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Get a single position by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) position = interface.getPosition(positionId) if not position: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") return position -@router.get("/positions/contract/{contractId}", response_model=List[TrusteePosition]) +@router.get("/{instanceId}/positions/contract/{contractId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") async def getPositionsByContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsByContract(contractId) -@router.get("/positions/organisation/{orgId}", response_model=List[TrusteePosition]) +@router.get("/{instanceId}/positions/organisation/{orgId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") async def getPositionsByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsByOrganisation(orgId) -@router.post("/positions", response_model=TrusteePosition, status_code=201) +@router.post("/{instanceId}/positions", response_model=TrusteePosition, status_code=201) @limiter.limit("10/minute") async def createPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePosition = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Create a new position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createPosition(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create position") return result -@router.put("/positions/{positionId}", response_model=TrusteePosition) +@router.put("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("10/minute") async def updatePosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), data: TrusteePosition = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Update a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -727,15 +888,18 @@ async def updatePosition( return result -@router.delete("/positions/{positionId}") +@router.delete("/{instanceId}/positions/{positionId}") @limiter.limit("10/minute") async def deletePosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -748,16 +912,19 @@ async def deletePosition( # ===== Position-Document Link Routes ===== -@router.get("/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) @limiter.limit("30/minute") async def getPositionDocuments( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePositionDocument]: """Get all position-document links with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositionDocuments(paginationParams) if paginationParams: @@ -775,69 +942,84 @@ async def getPositionDocuments( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/position-documents/{linkId}", response_model=TrusteePositionDocument) +@router.get("/{instanceId}/position-documents/{linkId}", response_model=TrusteePositionDocument) @limiter.limit("30/minute") async def getPositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Get a single position-document link by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) link = interface.getPositionDocument(linkId) if not link: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") return link -@router.get("/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") async def getDocumentsForPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all document links for a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getDocumentsForPosition(positionId) -@router.get("/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") async def getPositionsForDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all position links for a document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsForDocument(documentId) -@router.post("/position-documents", response_model=TrusteePositionDocument, status_code=201) +@router.post("/{instanceId}/position-documents", response_model=TrusteePositionDocument, status_code=201) @limiter.limit("10/minute") async def createPositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePositionDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Create a new position-document link.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createPositionDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create link") return result -@router.delete("/position-documents/{linkId}") +@router.delete("/{instanceId}/position-documents/{linkId}") @limiter.limit("10/minute") async def deletePositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position-document link.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPositionDocument(linkId) if not existing: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") diff --git a/modules/routes/routeFeatureAutomation.py b/modules/routes/routeFeatureWorkflow.py similarity index 95% rename from modules/routes/routeFeatureAutomation.py rename to modules/routes/routeFeatureWorkflow.py index ea58e271..ac002332 100644 --- a/modules/routes/routeFeatureAutomation.py +++ b/modules/routes/routeFeatureWorkflow.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext # Configure logger @@ -75,7 +75,7 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface + from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.features.workflow import syncAutomationEvents @@ -126,7 +126,7 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatObjects.getInterface(context.user) + chatInterface = interfaceDbChatbot.getInterface(context.user) automation = chatInterface.getAutomationDefinition(automation_id) if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index 3eef1980..53375849 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -204,12 +204,13 @@ async def exportUserData( for inv in invitationsUsed ] - # Audit log - audit_logger.logSecurityEvent( + # Audit log - GDPR Article 15 data export + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_data_export", - details="User requested data export (Article 15)" + details="User requested data export (GDPR Article 15 - Right of Access)", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} exported personal data (GDPR Art. 15)") @@ -304,12 +305,13 @@ async def exportPortableData( "about": portableData } - # Audit log - audit_logger.logSecurityEvent( + # Audit log - GDPR Article 20 data portability + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_data_portability", - details="User requested portable data export (Article 20)" + details="User requested portable data export (GDPR Article 20 - Right to Data Portability)", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} exported portable data (GDPR Art. 20)") @@ -431,12 +433,13 @@ async def deleteAccount( rootInterface.db.recordDelete(User, str(currentUser.id)) deletedData.append("User account deleted") - # Audit log (before user is deleted) - audit_logger.logSecurityEvent( + # Audit log (before user is deleted) - GDPR Article 17 account deletion + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_account_deletion", - details=f"User deleted own account (Article 17). Data: {', '.join(deletedData)}" + details=f"User deleted own account (GDPR Article 17 - Right to Erasure). Data: {', '.join(deletedData)}", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17)") diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 4e61a045..2a8f65fd 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -624,7 +624,10 @@ async def logout( userId=str(currentUser.id), mandateId="system", action="logout", - successInfo="google_auth_logout" + successInfo="google_auth_logout", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 4b64b671..8589db04 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -142,7 +142,10 @@ async def login( userId=str(user.id), mandateId="system", action="login", - successInfo="local_auth_success" + successInfo="local_auth_success", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails @@ -171,10 +174,13 @@ async def login( try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( - userId="unknown", - mandateId="unknown", - action="login", - successInfo=f"failed: {error_msg}" + userId=formData.username or "unknown", + mandateId="system", + action="login_failed", + successInfo=f"failed: {error_msg}", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=False ) except Exception: # Don't fail if audit logging fails @@ -438,7 +444,10 @@ async def logout(request: Request, response: Response, currentUser: User = Depen userId=str(currentUser.id), mandateId="system", action="logout", - successInfo=f"revoked_tokens: {revoked}" + successInfo=f"revoked_tokens: {revoked}", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index c145d1d3..77dc9885 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -634,7 +634,10 @@ async def logout( userId=str(currentUser.id), mandateId="system", action="logout", - successInfo="microsoft_auth_logout" + successInfo="microsoft_auth_logout", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 6d2f78ee..1c7d9e80 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -14,12 +14,12 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects -from modules.interfaces.interfaceDbChatObjects import getInterface +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +from modules.interfaces.interfaceDbChatbot import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( ChatWorkflow, ChatMessage, ChatLog, @@ -45,7 +45,7 @@ router = APIRouter( ) def getServiceChat(currentUser: User): - return interfaceDbChatObjects.getInterface(currentUser) + return interfaceDbChatbot.getInterface(currentUser) # Consolidated endpoint for getting all workflows @router.get("/", response_model=PaginatedResponse[ChatWorkflow]) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index 7033bfbb..b75b2454 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -3,7 +3,7 @@ from typing import Any, Optional from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow class PublicService: """Lightweight proxy exposing only public callable attributes of a target. @@ -49,7 +49,7 @@ class Services: # Initialize interfaces with explicit mandateId - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface + from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface @@ -58,7 +58,7 @@ class Services: from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbTrusteeObjects import getInterface as getTrusteeInterface + from modules.interfaces.interfaceDbTrustee import getInterface as getTrusteeInterface self.interfaceDbTrustee = getTrusteeInterface(user, mandateId=mandateId) # Expose RBAC directly on services for convenience diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index cd86c6a8..55bd2544 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -6,7 +6,7 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument +from modules.datamodels.datamodelChatbot import PromptPlaceholder, ChatDocument from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py index a866f68f..ec6a26d2 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py index 821851a4..e90ecfeb 100644 --- a/modules/services/serviceAi/subDocumentIntents.py +++ b/modules/services/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 137dcd05..0c82929d 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChatbot import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 13739dea..64678f54 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -11,7 +11,7 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index a49b78c7..adc0ea78 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -6,7 +6,7 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.services.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index d3d4dfda..5d3a8497 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -157,11 +157,11 @@ class UtilsService: def storeDebugMessageAndDocuments(self, message, currentUser): """ - Wrapper to store debug messages and documents via interfaceDbChatObjects. - Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatObjects. + Wrapper to store debug messages and documents via interfaceDbChatbot. + Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatbot. """ try: - from modules.interfaces.interfaceDbChatObjects import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChatbot import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/shared/auditLogger.py b/modules/shared/auditLogger.py index 46e80d5f..48cd8fdb 100644 --- a/modules/shared/auditLogger.py +++ b/modules/shared/auditLogger.py @@ -4,201 +4,471 @@ Audit Logging System for PowerOn Gateway This module provides centralized audit logging functionality for security events, -user actions, and system access patterns. +user actions, and system access patterns. Logs are stored in the database for +GDPR compliance and security monitoring. + +GDPR Requirements Addressed: +- Article 5(1)(f): Integrity and confidentiality - secure audit trail +- Article 17: Right to erasure - audit log retention with automatic cleanup +- Article 30: Records of processing activities - comprehensive event logging """ import logging -import os from datetime import datetime from typing import Optional, Dict, Any -from logging.handlers import RotatingFileHandler + from modules.shared.configuration import APP_CONFIG +from modules.shared.timeUtils import getUtcTimestamp - -class DailyRotatingFileHandler(RotatingFileHandler): - """ - A rotating file handler that automatically switches to a new file when the date changes. - The log file name includes the current date and switches at midnight. - """ - - def __init__(self, logDir, filenamePrefix, maxBytes=10485760, backupCount=5, **kwargs): - self.logDir = logDir - self.filenamePrefix = filenamePrefix - self.currentDate = None - self.currentFile = None - - # Initialize with today's file - self._updateFileIfNeeded() - - # Call parent constructor with current file - super().__init__(self.currentFile, maxBytes=maxBytes, backupCount=backupCount, **kwargs) - - def _updateFileIfNeeded(self): - """Update the log file if the date has changed""" - today = datetime.now().strftime("%Y%m%d") - - if self.currentDate != today: - self.currentDate = today - newFile = os.path.join(self.logDir, f"{self.filenamePrefix}_{today}.log") - - if self.currentFile != newFile: - self.currentFile = newFile - return True - return False - - def emit(self, record): - """Emit a log record, switching files if date has changed""" - # Check if we need to switch to a new file - if self._updateFileIfNeeded(): - # Close current file and open new one - if self.stream: - self.stream.close() - self.stream = None - - # Update the baseFilename for the parent class - self.baseFilename = self.currentFile - # Reopen the stream - if not self.delay: - self.stream = self._open() - - # Call parent emit method - super().emit(record) +logger = logging.getLogger(__name__) class AuditLogger: - """Centralized audit logging system""" + """ + Centralized audit logging system with database storage. + + Logs security-relevant events to PostgreSQL for: + - GDPR compliance + - Security monitoring + - Access tracking + - Incident investigation + """ def __init__(self): - self.logger = None - self._setupAuditLogger() - - def _setupAuditLogger(self): - """Setup the audit logger with daily file rotation""" + self._db = None + self._modelClass = None + self._initialized = False + self._fallbackToStdout = True + + def _ensureInitialized(self) -> bool: + """Lazily initialize database connection to avoid circular imports.""" + if self._initialized: + return self._db is not None + + self._initialized = True + try: - # Get log directory from config - logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") - if not os.path.isabs(logDir): - # If relative path, make it relative to the gateway directory - gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - logDir = os.path.join(gatewayDir, logDir) + from modules.datamodels.datamodelAudit import AuditLogEntry + from modules.connectors.connectorDbPostgre import DatabaseConnector - # Ensure log directory exists - os.makedirs(logDir, exist_ok=True) + self._modelClass = AuditLogEntry - # Create audit logger - self.logger = logging.getLogger('audit') - self.logger.setLevel(logging.INFO) + # Get database configuration + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_app" # Store audit logs in the main app database + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) - # Remove any existing handlers to avoid duplicates - for handler in self.logger.handlers[:]: - self.logger.removeHandler(handler) - - # Create daily rotating file handler for audit log - rotationSize = int(APP_CONFIG.get("APP_LOGGING_ROTATION_SIZE", 10485760)) # Default: 10MB - backupCount = int(APP_CONFIG.get("APP_LOGGING_BACKUP_COUNT", 5)) - - fileHandler = DailyRotatingFileHandler( - logDir=logDir, - filenamePrefix="log_audit", - maxBytes=rotationSize, - backupCount=backupCount + # Create database connector with system user context + self._db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId="system" # Audit logs are created by system ) - # Create formatter for audit log - auditFormatter = logging.Formatter( - fmt="%(asctime)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" - ) - fileHandler.setFormatter(auditFormatter) + # Initialize database and ensure table exists + self._db.initDbSystem() + self._db._ensureTableExists(AuditLogEntry) - # Add handler to logger - self.logger.addHandler(fileHandler) - - # Prevent propagation to root logger - self.logger.propagate = False + logger.info("AuditLogger database connection initialized successfully") + return True except Exception as e: - # Fallback to standard logger if audit setup fails - self.logger = logging.getLogger(__name__) - self.logger.error(f"Failed to setup audit logger: {str(e)}") + logger.warning(f"AuditLogger database initialization failed, using fallback logging: {e}") + self._db = None + return False - def logEvent(self, - userId: str, - mandateId: str, - category: str, - action: str, - details: str = "", - timestamp: Optional[datetime] = None) -> None: + def _logToFallback(self, entry: Dict[str, Any]) -> None: + """Log to standard logger as fallback when database is unavailable.""" + if self._fallbackToStdout: + fallbackMsg = ( + f"AUDIT | {entry.get('timestamp', '')} | " + f"{entry.get('userId', '')} | {entry.get('mandateId', '')} | " + f"{entry.get('category', '')} | {entry.get('action', '')} | " + f"{entry.get('details', '')}" + ) + logging.getLogger('audit.fallback').info(fallbackMsg) + + def logEvent( + self, + userId: str, + mandateId: Optional[str] = None, + category: str = "system", + action: str = "", + details: str = "", + featureInstanceId: Optional[str] = None, + resourceType: Optional[str] = None, + resourceId: Optional[str] = None, + ipAddress: Optional[str] = None, + userAgent: Optional[str] = None, + success: bool = True, + errorMessage: Optional[str] = None, + username: Optional[str] = None, + timestamp: Optional[float] = None + ) -> Optional[str]: """ - Log an audit event + Log an audit event to the database. Args: - userId: User identifier - mandateId: Mandate identifier (can be empty if not applicable) - category: Event category (e.g., 'key', 'access', 'data') - action: Specific action (e.g., 'decode', 'login', 'logout') + userId: User identifier (or 'system' for system events) + mandateId: Mandate context (can be None for system-level events) + category: Event category (access, key, data, security, gdpr, permission, system) + action: Specific action performed details: Additional details about the event + featureInstanceId: Feature instance context (if applicable) + resourceType: Type of resource affected + resourceId: ID of the affected resource + ipAddress: Client IP address + userAgent: Client user agent + success: Whether the action was successful + errorMessage: Error message if action failed + username: Username at the time of event (for historical reference) timestamp: Optional custom timestamp (defaults to current time) + + Returns: + ID of the created audit log entry, or None if logging failed """ try: - if not self.logger: - return - - # Use provided timestamp or current time - if timestamp is None: - timestamp = datetime.now() - - # Format the audit log entry - # Format: timestamp | userid | mandateid | category | action | details - auditEntry = f"{userId} | {mandateId} | {category} | {action} | {details}" - - # Log the event - self.logger.info(auditEntry) + # Prepare the entry data + entryData = { + "timestamp": timestamp if timestamp else getUtcTimestamp(), + "userId": userId or "unknown", + "username": username, + "mandateId": mandateId, + "featureInstanceId": featureInstanceId, + "category": category, + "action": action, + "resourceType": resourceType, + "resourceId": resourceId, + "details": details if details else None, + "ipAddress": ipAddress, + "userAgent": userAgent, + "success": success, + "errorMessage": errorMessage + } + # Try to write to database + if self._ensureInitialized() and self._db: + from modules.datamodels.datamodelAudit import AuditLogEntry + + entry = AuditLogEntry(**entryData) + created = self._db.recordCreate(AuditLogEntry, entry.model_dump()) + + if created and created.get("id"): + return created["id"] + else: + self._logToFallback(entryData) + return None + else: + # Use fallback logging + self._logToFallback(entryData) + return None + except Exception as e: - # Use standard logger as fallback - logging.getLogger(__name__).error(f"Failed to log audit event: {str(e)}") + logger.error(f"Failed to log audit event: {e}") + # Try fallback + try: + self._logToFallback(entryData) + except Exception: + pass + return None - def logKeyAccess(self, userId: str, mandateId: str, keyName: str, action: str) -> None: - """Log key access events (decode/encode)""" - self.logEvent( + # ===== Convenience Methods for Common Event Types ===== + + def logKeyAccess( + self, + userId: str, + mandateId: str, + keyName: str, + action: str, + ipAddress: Optional[str] = None + ) -> Optional[str]: + """Log key access events (encode/decode).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="key", action=action, - details=keyName + details=f"Key: {keyName}", + resourceType="EncryptionKey", + resourceId=keyName, + ipAddress=ipAddress ) - def logUserAccess(self, userId: str, mandateId: str, action: str, successInfo: str = "") -> None: - """Log user access events (login/logout)""" - self.logEvent( + def logUserAccess( + self, + userId: str, + mandateId: str, + action: str, + successInfo: str = "", + ipAddress: Optional[str] = None, + userAgent: Optional[str] = None, + success: bool = True + ) -> Optional[str]: + """Log user access events (login/logout).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="access", action=action, - details=successInfo + details=successInfo, + ipAddress=ipAddress, + userAgent=userAgent, + success=success ) - def logDataAccess(self, userId: str, mandateId: str, action: str, details: str = "") -> None: - """Log data access events""" - self.logEvent( + def logDataAccess( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + resourceType: Optional[str] = None, + resourceId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ) -> Optional[str]: + """Log data access events (CRUD operations).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="data", action=action, - details=details + details=details, + resourceType=resourceType, + resourceId=resourceId, + featureInstanceId=featureInstanceId ) - def logSecurityEvent(self, userId: str, mandateId: str, action: str, details: str = "") -> None: - """Log security-related events""" - self.logEvent( + def logSecurityEvent( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + ipAddress: Optional[str] = None, + success: bool = True, + errorMessage: Optional[str] = None + ) -> Optional[str]: + """Log security-related events.""" + return self.logEvent( userId=userId, mandateId=mandateId, category="security", action=action, - details=details + details=details, + ipAddress=ipAddress, + success=success, + errorMessage=errorMessage ) + + def logGdprEvent( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + ipAddress: Optional[str] = None + ) -> Optional[str]: + """Log GDPR-specific events (data export, deletion, etc.).""" + return self.logEvent( + userId=userId, + mandateId=mandateId, + category="gdpr", + action=action, + details=details, + ipAddress=ipAddress + ) + + def logPermissionChange( + self, + userId: str, + mandateId: str, + action: str, + targetUserId: str, + details: str = "", + resourceType: Optional[str] = None, + resourceId: Optional[str] = None + ) -> Optional[str]: + """Log permission/role changes.""" + return self.logEvent( + userId=userId, + mandateId=mandateId, + category="permission", + action=action, + details=f"Target user: {targetUserId}. {details}", + resourceType=resourceType, + resourceId=resourceId + ) + + # ===== Audit Log Query Methods ===== + + def getAuditLogs( + self, + userId: Optional[str] = None, + mandateId: Optional[str] = None, + category: Optional[str] = None, + action: Optional[str] = None, + fromTimestamp: Optional[float] = None, + toTimestamp: Optional[float] = None, + limit: int = 100 + ) -> list: + """ + Query audit logs from database. + + Args: + userId: Filter by user ID + mandateId: Filter by mandate ID + category: Filter by category + action: Filter by action + fromTimestamp: Filter events after this timestamp + toTimestamp: Filter events before this timestamp + limit: Maximum number of records to return + + Returns: + List of audit log entries + """ + if not self._ensureInitialized() or not self._db: + return [] + + try: + from modules.datamodels.datamodelAudit import AuditLogEntry + + # Build filter + recordFilter = {} + if userId: + recordFilter["userId"] = userId + if mandateId: + recordFilter["mandateId"] = mandateId + if category: + recordFilter["category"] = category + if action: + recordFilter["action"] = action + + # Query database + records = self._db.getRecordset( + AuditLogEntry, + recordFilter=recordFilter if recordFilter else None, + orderBy="timestamp DESC" + ) + + # Apply timestamp filtering in Python (PostgreSQL connector may not support $gt/$lt) + if fromTimestamp or toTimestamp: + filteredRecords = [] + for record in records: + ts = record.get("timestamp", 0) + if fromTimestamp and ts < fromTimestamp: + continue + if toTimestamp and ts > toTimestamp: + continue + filteredRecords.append(record) + records = filteredRecords + + # Apply limit + return records[:limit] + + except Exception as e: + logger.error(f"Failed to query audit logs: {e}") + return [] + + # ===== Cleanup Methods ===== + + def cleanupOldEntries(self, retentionDays: int = 365) -> int: + """ + Remove audit log entries older than the retention period. + + GDPR Note: Audit logs should be retained for a reasonable period + for security and compliance purposes, but not indefinitely. + Default retention is 1 year (365 days). + + Args: + retentionDays: Number of days to retain audit logs + + Returns: + Number of entries deleted + """ + if not self._ensureInitialized() or not self._db: + logger.warning("Cannot cleanup audit logs: database not initialized") + return 0 + + try: + from modules.datamodels.datamodelAudit import AuditLogEntry + import time + + # Calculate cutoff timestamp + cutoffTimestamp = time.time() - (retentionDays * 24 * 60 * 60) + + # Query old entries + allRecords = self._db.getRecordset(AuditLogEntry) + oldRecords = [r for r in allRecords if r.get("timestamp", 0) < cutoffTimestamp] + + # Delete old entries + deletedCount = 0 + for record in oldRecords: + recordId = record.get("id") + if recordId: + if self._db.recordDelete(AuditLogEntry, recordId): + deletedCount += 1 + + logger.info(f"Audit log cleanup: removed {deletedCount} entries older than {retentionDays} days") + + # Log the cleanup action itself + self.logEvent( + userId="system", + mandateId="system", + category="system", + action="audit_cleanup", + details=f"Removed {deletedCount} entries older than {retentionDays} days" + ) + + return deletedCount + + except Exception as e: + logger.error(f"Failed to cleanup audit logs: {e}") + return 0 # Global audit logger instance audit_logger = AuditLogger() + + +# ===== Scheduler Integration ===== + +async def runAuditLogCleanup() -> None: + """ + Scheduled task to cleanup old audit log entries. + Called by the event scheduler. + """ + try: + retentionDays = int(APP_CONFIG.get("AUDIT_LOG_RETENTION_DAYS", 365)) + deletedCount = audit_logger.cleanupOldEntries(retentionDays=retentionDays) + logger.info(f"Scheduled audit log cleanup completed: {deletedCount} entries removed") + except Exception as e: + logger.error(f"Scheduled audit log cleanup failed: {e}") + + +def registerAuditLogCleanupScheduler() -> None: + """ + Register the audit log cleanup job with the event scheduler. + Should be called during application startup. + """ + try: + from modules.shared.eventManagement import eventManager + + # Run cleanup daily at 3 AM + eventManager.registerCron( + jobId="audit_log_cleanup", + func=runAuditLogCleanup, + cronKwargs={ + "hour": "3", + "minute": "0" + } + ) + + logger.info("Audit log cleanup scheduler registered (daily at 03:00)") + + except Exception as e: + logger.error(f"Failed to register audit log cleanup scheduler: {e}") diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index 39d6e16f..a3f45261 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 4f9bbd21..77cb361f 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 65e95a32..6c509a9e 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index f804c0b9..bddeb252 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index e32c1965..806679df 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index bb6f8437..96f2609c 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 62b43bce..4c5d8314 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index ff7e896f..c6e6d560 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index 5b90ce13..cccad45d 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index 9991285b..fedbc46b 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 8e3b7185..0b646251 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index 2f011a25..015eb1e3 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index 45b60cad..f00192f6 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index cbec7960..1e44b1cb 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index 631795b3..afa0c5fc 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 55d99654..6e6106b0 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index b997889e..d72e3d55 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 2bd7ab74..49ddece2 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index bbdc2cc7..24c097e8 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index 5ac4e548..adfe13ea 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 59604896..06f26e89 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 2d325d9f..4ff700ca 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index f8831d59..4531859f 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index 9b7fb011..da9f8cd4 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index a4bf18b6..05997512 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index f149e482..287612ff 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index c64a6637..e6c2a276 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 722dbc99..4eac8544 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index 62b6dd94..a9b837aa 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index 318271c3..d0838633 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index 73cdb730..eaf3254f 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index e9361853..ddce6206 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 1f469b80..85d7b123 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 0e4d6ee4..7bde4da7 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ActionResult, ActionItem, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index a4ae05e9..0daac228 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 0fac427c..4b1fcad5 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.datamodels.datamodelChat import WorkflowModeEnum + from modules.datamodels.datamodelChatbot import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index e3131939..85f1f824 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index 770c868a..e8837d65 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskResult, ActionItem +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index f7754eab..b821511b 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.datamodels.datamodelChat import ReviewContext + from modules.datamodels.datamodelChatbot import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.datamodels.datamodelChat import ReviewResult + from modules.datamodels.datamodelChatbot import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 1cdf0d53..b0186be9 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation +from modules.datamodels.datamodelChatbot import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 0be4e029..30e9af4d 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.datamodels.datamodelChatbot import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.datamodels.datamodelChatbot import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict @@ -452,10 +452,10 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceDbAppObjects import getRootInterface rootInterface = getRootInterface() - interfaceDbChat = interfaceDbChatObjects.getInterface(rootInterface.currentUser) + interfaceDbChat = interfaceDbChatbot.getInterface(rootInterface.currentUser) # Get workflow logs chatData = interfaceDbChat.getUnifiedChatData(context.workflow.id, None) diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 31878033..10932529 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index 11a54ca1..e1d767c4 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 9c9d6c84..317a6cb7 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -6,9 +6,9 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING -from modules.datamodels import datamodelChat -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum +from modules.datamodels import datamodelChatbot +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -102,7 +102,7 @@ class WorkflowProcessor: self.services.chat.progressLogFinish(operationId, False) raise - async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChat.TaskResult: + async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChatbot.TaskResult: """Execute a task step using the appropriate mode""" import time @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.datamodels.datamodelChat import ActionDocument + from modules.datamodels.datamodelChatbot import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.datamodels.datamodelChat import ChatMessage, ChatDocument, ActionDocument + from modules.datamodels.datamodelChatbot import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index a9b656eb..e6ecdbbd 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.datamodels.datamodelChat import TaskContext +from modules.datamodels.datamodelChatbot import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.datamodels.datamodelChat import ActionResult + from modules.datamodels.datamodelChatbot import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 12a374f8..94eb6158 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -85,7 +85,7 @@ class AIModelsTester: self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index 36a8505a..dd5d68e3 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -94,8 +94,8 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) # Import and initialize services from modules.services import getInterface as getServices @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.datamodels.datamodelChat import ChatMessage + from modules.datamodels.datamodelChatbot import ChatMessage import time testMessage = ChatMessage( @@ -201,8 +201,8 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,8 +283,8 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 51059745..478e9baf 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,10 +42,10 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot currentTimestamp = time.time() @@ -67,7 +67,7 @@ class AIBehaviorTester: ) # SAVE workflow to database so it exists for access control - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 7ed171c0..3a0cf2a3 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,10 +20,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class WorkflowWithDocumentsTester: @@ -192,7 +192,7 @@ class WorkflowWithDocumentsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) if not currentWorkflow: @@ -225,7 +225,7 @@ class WorkflowWithDocumentsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index c65fa401..784b1756 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,10 +22,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class WorkflowPromptVariationsTester: @@ -115,7 +115,7 @@ class WorkflowPromptVariationsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(workflow.id) if not currentWorkflow: @@ -140,7 +140,7 @@ class WorkflowPromptVariationsTester: def _analyzeWorkflowResults(self, workflow: Any) -> Dict[str, Any]: """Analyze workflow results and extract information.""" - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(workflow.id) if not workflow: diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 3e33c996..9c96d95f 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class DocumentGenerationFormatsTester: @@ -251,7 +251,7 @@ class DocumentGenerationFormatsTester: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -296,7 +296,7 @@ class DocumentGenerationFormatsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 9ce9b367..a0b744f4 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class DocumentGenerationFormatsTester10: @@ -248,7 +248,7 @@ class DocumentGenerationFormatsTester10: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -293,7 +293,7 @@ class DocumentGenerationFormatsTester10: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 266b27e5..e1331cd1 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,10 +23,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class CodeGenerationFormatsTester11: @@ -190,7 +190,7 @@ class CodeGenerationFormatsTester11: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -235,7 +235,7 @@ class CodeGenerationFormatsTester11: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index a2b69576..9409a9e6 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index ae502397..b91aa1e7 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 09f6e92c..25a5af8e 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel diff --git a/tool_db_export_migration.py b/tool_db_export_migration.py new file mode 100644 index 00000000..09aa2f8f --- /dev/null +++ b/tool_db_export_migration.py @@ -0,0 +1,508 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Datenbank Export-Tool für Migration. + +Dieses Script exportiert alle Daten aus ALLEN PowerOn PostgreSQL-Datenbanken +in eine JSON-Datei, die als Migrationsdatensatz verwendet werden kann. + +Datenbanken: + - poweron_app (User, Mandate, RBAC, Features, etc.) + - poweron_chat (Chat-Konversationen und Nachrichten) + - poweron_management (Workflows, Prompts, Connections, etc.) + - poweron_realestate (Real Estate Daten) + - poweron_trustee (Trustee Daten) + +Verwendung: + python tool_db_export_migration.py [--output ] [--pretty] + +Optionen: + --output, -o Pfad zur Ausgabedatei (Standard: migration_export_.json) + --pretty, -p JSON formatiert ausgeben (für bessere Lesbarkeit) + --exclude Komma-getrennte Liste von Tabellen, die ausgeschlossen werden sollen + --include-meta System-Metadaten (_createdAt, _modifiedAt, etc.) beibehalten + --db Nur bestimmte Datenbank(en) exportieren (komma-getrennt) +""" + +import os +import sys +import json +import argparse +import logging +from datetime import datetime +from typing import Dict, List, Any, Optional +from pathlib import Path + +import psycopg2 +import psycopg2.extras + +# Logging konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +# Alle PowerOn Datenbanken +ALL_DATABASES = [ + "poweron_app", # Haupt-App: User, Mandate, RBAC, Features + "poweron_chat", # Chat-Konversationen + "poweron_management", # Workflows, Prompts, Connections + "poweron_realestate", # Real Estate + "poweron_trustee", # Trustee +] + + +def _loadEnvConfig() -> Dict[str, str]: + """Lädt die Konfiguration direkt aus der .env Datei.""" + config = {} + envPath = Path(__file__).parent / '.env' + + if not envPath.exists(): + logger.warning(f"Environment file not found at {envPath}") + return config + + # Versuche verschiedene Encodings + encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] + + for encoding in encodings: + try: + with open(envPath, 'r', encoding=encoding) as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' in line: + key, value = line.split('=', 1) + config[key.strip()] = value.strip() + # Erfolgreich geladen + return config + except UnicodeDecodeError: + continue + except Exception as e: + logger.error(f"Error loading .env file with {encoding}: {e}") + continue + + logger.error(f"Could not load .env file with any encoding") + return config + + +# Globale Konfiguration laden +_ENV_CONFIG = _loadEnvConfig() + + +def _getConfigValue(key: str, default: str = None) -> str: + """Holt einen Konfigurationswert.""" + return _ENV_CONFIG.get(key, os.environ.get(key, default)) + + +def _databaseExists(dbDatabase: str) -> bool: + """Prüft ob eine Datenbank existiert.""" + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + # Verbinde zur postgres Datenbank um zu prüfen + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database="postgres", + user=dbUser, + password=dbPassword + ) + conn.autocommit = True + + with conn.cursor() as cursor: + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (dbDatabase,) + ) + exists = cursor.fetchone() is not None + + conn.close() + return exists + except Exception as e: + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + return False + + +def _getDbConnection(dbDatabase: str): + """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht - übersprungen") + return None + + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database=dbDatabase, + user=dbUser, + password=dbPassword, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.set_client_encoding('UTF8') + return conn + except Exception as e: + logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + raise + + +def _getTables(conn) -> List[str]: + """Gibt alle Tabellennamen in der Datenbank zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """) + tables = [row["table_name"] for row in cursor.fetchall()] + return tables + + +def _getTableData(conn, tableName: str, includeMeta: bool = False) -> List[Dict[str, Any]]: + """Liest alle Daten aus einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'SELECT * FROM "{tableName}"') + rows = cursor.fetchall() + + records = [] + for row in rows: + record = dict(row) + + # Optional: System-Metadaten entfernen + if not includeMeta: + metaFields = ["_createdAt", "_modifiedAt", "_createdBy", "_modifiedBy"] + for field in metaFields: + record.pop(field, None) + + # Konvertiere JSONB-Felder (sind bereits als Dict/List von psycopg2) + for key, value in record.items(): + if isinstance(value, (int, float)): + record[key] = float(value) if isinstance(value, float) else int(value) + + records.append(record) + + return records + + +def _getTableRowCount(conn, tableName: str) -> int: + """Zählt die Anzahl der Zeilen in einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'SELECT COUNT(*) as count FROM "{tableName}"') + result = cursor.fetchone() + return result["count"] if result else 0 + + +def _exportSingleDatabase( + dbDatabase: str, + excludeTables: List[str], + includeMeta: bool +) -> Optional[Dict[str, Any]]: + """Exportiert eine einzelne Datenbank.""" + conn = _getDbConnection(dbDatabase) + + if conn is None: + return None + + try: + allTables = _getTables(conn) + + # System-Tabellen ausschliessen + systemTables = ["_system"] + tablesToExport = [ + t for t in allTables + if t not in systemTables and t not in excludeTables + ] + + dbExport = { + "tables": {}, + "summary": {}, + "tableCount": len(tablesToExport), + "totalRecords": 0 + } + + for tableName in tablesToExport: + try: + records = _getTableData(conn, tableName, includeMeta) + rowCount = len(records) + dbExport["totalRecords"] += rowCount + + dbExport["tables"][tableName] = records + dbExport["summary"][tableName] = {"recordCount": rowCount} + + if rowCount > 0: + logger.info(f" {tableName}: {rowCount} Datensätze") + + except Exception as e: + logger.error(f" Fehler bei Tabelle {tableName}: {e}") + dbExport["tables"][tableName] = [] + dbExport["summary"][tableName] = {"recordCount": 0, "error": str(e)} + + return dbExport + + finally: + conn.close() + + +def exportDatabase( + outputPath: Optional[str] = None, + prettyPrint: bool = False, + excludeTables: Optional[List[str]] = None, + includeMeta: bool = False, + onlyDatabases: Optional[List[str]] = None +) -> str: + """ + Exportiert alle Datenbanken in eine JSON-Datei. + + Args: + outputPath: Pfad zur Ausgabedatei (optional) + prettyPrint: JSON formatiert ausgeben + excludeTables: Liste von Tabellen, die ausgeschlossen werden sollen + includeMeta: System-Metadaten beibehalten + onlyDatabases: Nur diese Datenbanken exportieren + + Returns: + Pfad zur erstellten Exportdatei + """ + excludeTables = excludeTables or [] + + # Welche Datenbanken exportieren? + databasesToExport = onlyDatabases if onlyDatabases else ALL_DATABASES + + # Standard-Ausgabepfad generieren (im Log-Ordner) + if not outputPath: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + logDir = _getConfigValue("APP_LOGGING_LOG_DIR") + if logDir and os.path.isabs(logDir): + outputDir = logDir + else: + outputDir = os.path.join(os.path.dirname(__file__), "local", "logs") + os.makedirs(outputDir, exist_ok=True) + outputPath = os.path.join(outputDir, f"migration_export_{timestamp}.json") + + logger.info(f"Starte Export von {len(databasesToExport)} Datenbank(en)...") + logger.info(f"Datenbanken: {', '.join(databasesToExport)}") + + # Export-Struktur erstellen + exportData = { + "meta": { + "exportedAt": datetime.utcnow().isoformat() + "Z", + "exportedFrom": _getConfigValue("APP_ENV_LABEL", "unknown"), + "version": "1.0", + "databaseCount": 0, + "totalTables": 0, + "totalRecords": 0, + "excludedTables": excludeTables, + "includesMeta": includeMeta + }, + "databases": {} + } + + # Jede Datenbank exportieren + for dbName in databasesToExport: + logger.info(f"Exportiere Datenbank: {dbName}") + + dbExport = _exportSingleDatabase(dbName, excludeTables, includeMeta) + + if dbExport is not None: + exportData["databases"][dbName] = dbExport + exportData["meta"]["databaseCount"] += 1 + exportData["meta"]["totalTables"] += dbExport["tableCount"] + exportData["meta"]["totalRecords"] += dbExport["totalRecords"] + logger.info(f" -> {dbExport['tableCount']} Tabellen, {dbExport['totalRecords']} Datensätze") + else: + logger.info(f" -> Übersprungen (existiert nicht)") + + # JSON-Datei schreiben + logger.info(f"Schreibe Exportdatei: {outputPath}") + + with open(outputPath, "w", encoding="utf-8") as f: + if prettyPrint: + json.dump(exportData, f, indent=2, ensure_ascii=False, default=str) + else: + json.dump(exportData, f, ensure_ascii=False, default=str) + + # Dateigrösse berechnen + fileSize = os.path.getsize(outputPath) + fileSizeStr = _formatFileSize(fileSize) + + logger.info(f"Export abgeschlossen!") + logger.info(f" Datenbanken: {exportData['meta']['databaseCount']}") + logger.info(f" Tabellen: {exportData['meta']['totalTables']}") + logger.info(f" Datensätze: {exportData['meta']['totalRecords']}") + logger.info(f" Dateigrösse: {fileSizeStr}") + logger.info(f" Ausgabedatei: {outputPath}") + + return outputPath + + +def _formatFileSize(sizeBytes: int) -> str: + """Formatiert Dateigrösse in lesbares Format.""" + for unit in ['B', 'KB', 'MB', 'GB']: + if sizeBytes < 1024: + return f"{sizeBytes:.2f} {unit}" + sizeBytes /= 1024 + return f"{sizeBytes:.2f} TB" + + +def printDatabaseSummary(): + """Zeigt eine Zusammenfassung aller Datenbanken an.""" + print("\n" + "=" * 70) + print("DATENBANK ZUSAMMENFASSUNG - ALLE POWEREON DATENBANKEN") + print("=" * 70) + print(f"Umgebung: {_getConfigValue('APP_ENV_LABEL', 'unknown')}") + print(f"Host: {_getConfigValue('DB_HOST', 'localhost')}") + print("=" * 70) + + grandTotalRecords = 0 + grandTotalTables = 0 + + for dbName in ALL_DATABASES: + print(f"\n{dbName}") + print("-" * 70) + + conn = _getDbConnection(dbName) + if conn is None: + print(" (Datenbank existiert nicht)") + continue + + try: + tables = _getTables(conn) + dbTotalRecords = 0 + + print(f" {'Tabelle':<45} {'Datensätze':>15}") + print(f" {'-' * 45} {'-' * 15}") + + for tableName in tables: + if tableName.startswith("_"): + continue # System-Tabellen überspringen + count = _getTableRowCount(conn, tableName) + dbTotalRecords += count + if count > 0: # Nur nicht-leere Tabellen anzeigen + print(f" {tableName:<45} {count:>15}") + + print(f" {'-' * 45} {'-' * 15}") + print(f" {'Gesamt':<45} {dbTotalRecords:>15}") + + grandTotalRecords += dbTotalRecords + grandTotalTables += len([t for t in tables if not t.startswith("_")]) + + finally: + conn.close() + + print("\n" + "=" * 70) + print(f"GESAMTÜBERSICHT") + print(f" Datenbanken: {len(ALL_DATABASES)}") + print(f" Tabellen: {grandTotalTables}") + print(f" Datensätze: {grandTotalRecords}") + print("=" * 70 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Exportiert alle PowerOn Datenbank-Daten für Migration", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Datenbanken: + poweron_app - User, Mandate, RBAC, Features + poweron_chat - Chat-Konversationen + poweron_management - Workflows, Prompts, Connections + poweron_realestate - Real Estate Daten + poweron_trustee - Trustee Daten + +Beispiele: + python tool_db_export_migration.py + python tool_db_export_migration.py --pretty + python tool_db_export_migration.py -o backup.json --pretty + python tool_db_export_migration.py --db poweron_app,poweron_chat + python tool_db_export_migration.py --exclude Token,AuthEvent --include-meta + python tool_db_export_migration.py --summary + """ + ) + + parser.add_argument( + "-o", "--output", + help="Pfad zur Ausgabedatei", + type=str, + default=None + ) + + parser.add_argument( + "-p", "--pretty", + help="JSON formatiert ausgeben", + action="store_true" + ) + + parser.add_argument( + "--exclude", + help="Komma-getrennte Liste von Tabellen zum Ausschliessen", + type=str, + default="" + ) + + parser.add_argument( + "--include-meta", + help="System-Metadaten (_createdAt, etc.) beibehalten", + action="store_true" + ) + + parser.add_argument( + "--db", + help="Nur bestimmte Datenbank(en) exportieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--summary", + help="Nur Zusammenfassung anzeigen (kein Export)", + action="store_true" + ) + + args = parser.parse_args() + + # Nur Zusammenfassung anzeigen + if args.summary: + printDatabaseSummary() + return + + # Exclude-Liste parsen + excludeTables = [] + if args.exclude: + excludeTables = [t.strip() for t in args.exclude.split(",") if t.strip()] + + # Datenbank-Liste parsen + onlyDatabases = None + if args.db: + onlyDatabases = [db.strip() for db in args.db.split(",") if db.strip()] + + # Export durchführen + try: + outputPath = exportDatabase( + outputPath=args.output, + prettyPrint=args.pretty, + excludeTables=excludeTables, + includeMeta=args.include_meta, + onlyDatabases=onlyDatabases + ) + print(f"\n Export erfolgreich: {outputPath}\n") + + except Exception as e: + logger.error(f"Export fehlgeschlagen: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tool_db_import_migration.py b/tool_db_import_migration.py new file mode 100644 index 00000000..1ab4e4fe --- /dev/null +++ b/tool_db_import_migration.py @@ -0,0 +1,612 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Datenbank Import-Tool für Migration. + +Dieses Script importiert Daten aus einer JSON-Migrationsdatei +in ALLE PowerOn PostgreSQL-Datenbanken. + +ACHTUNG: Dieses Script kann bestehende Daten überschreiben! +Bitte vor dem Import ein Backup erstellen. + +Datenbanken: + - poweron_app (User, Mandate, RBAC, Features, etc.) + - poweron_chat (Chat-Konversationen und Nachrichten) + - poweron_management (Workflows, Prompts, Connections, etc.) + - poweron_realestate (Real Estate Daten) + - poweron_trustee (Trustee Daten) + +Verwendung: + python tool_db_import_migration.py [--dry-run] [--force] + +Optionen: + --dry-run Simuliert den Import ohne Änderungen + --force Bestätigung überspringen + --clear-first Tabellen vor dem Import leeren + --only-tables Komma-getrennte Liste von Tabellen (nur diese importieren) + --only-db Komma-getrennte Liste von Datenbanken (nur diese importieren) +""" + +import os +import sys +import json +import argparse +import logging +import time +from datetime import datetime +from typing import Dict, List, Any, Optional +from pathlib import Path + +import psycopg2 +import psycopg2.extras + +# Logging konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +# Alle PowerOn Datenbanken +ALL_DATABASES = [ + "poweron_app", + "poweron_chat", + "poweron_management", + "poweron_realestate", + "poweron_trustee", +] + + +def _loadEnvConfig() -> Dict[str, str]: + """Lädt die Konfiguration direkt aus der .env Datei.""" + config = {} + envPath = Path(__file__).parent / '.env' + + if not envPath.exists(): + logger.warning(f"Environment file not found at {envPath}") + return config + + # Versuche verschiedene Encodings + encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] + + for encoding in encodings: + try: + with open(envPath, 'r', encoding=encoding) as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' in line: + key, value = line.split('=', 1) + config[key.strip()] = value.strip() + # Erfolgreich geladen + return config + except UnicodeDecodeError: + continue + except Exception as e: + logger.error(f"Error loading .env file with {encoding}: {e}") + continue + + logger.error(f"Could not load .env file with any encoding") + return config + + +# Globale Konfiguration laden +_ENV_CONFIG = _loadEnvConfig() + + +def _getConfigValue(key: str, default: str = None) -> str: + """Holt einen Konfigurationswert.""" + return _ENV_CONFIG.get(key, os.environ.get(key, default)) + + +def _getUtcTimestamp() -> float: + """Gibt den aktuellen UTC-Timestamp zurück.""" + return time.time() + + +def _databaseExists(dbDatabase: str) -> bool: + """Prüft ob eine Datenbank existiert.""" + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database="postgres", + user=dbUser, + password=dbPassword + ) + conn.autocommit = True + + with conn.cursor() as cursor: + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (dbDatabase,) + ) + exists = cursor.fetchone() is not None + + conn.close() + return exists + except Exception as e: + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + return False + + +def _getDbConnection(dbDatabase: str, autocommit: bool = False): + """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht") + return None + + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database=dbDatabase, + user=dbUser, + password=dbPassword, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.set_client_encoding('UTF8') + conn.autocommit = autocommit + return conn + except Exception as e: + logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + raise + + +def _getExistingTables(conn) -> List[str]: + """Gibt alle Tabellennamen in der Datenbank zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """) + tables = [row["table_name"] for row in cursor.fetchall()] + return tables + + +def _getTableColumns(conn, tableName: str) -> List[str]: + """Gibt alle Spalten einer Tabelle zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = %s AND table_schema = 'public' + """, (tableName,)) + columns = [row["column_name"] for row in cursor.fetchall()] + return columns + + +def _clearTable(conn, tableName: str): + """Löscht alle Daten aus einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'DELETE FROM "{tableName}"') + + +def _insertRecord(conn, tableName: str, record: Dict[str, Any], existingColumns: List[str]) -> bool: + """Fügt einen Datensatz in eine Tabelle ein (UPSERT).""" + filteredRecord = {k: v for k, v in record.items() if k in existingColumns} + + if not filteredRecord: + return False + + # Metadaten hinzufügen falls nicht vorhanden + currentTime = _getUtcTimestamp() + if "_createdAt" not in filteredRecord and "_createdAt" in existingColumns: + filteredRecord["_createdAt"] = currentTime + if "_modifiedAt" in existingColumns: + filteredRecord["_modifiedAt"] = currentTime + + columns = list(filteredRecord.keys()) + values = [] + + for col in columns: + value = filteredRecord[col] + if isinstance(value, (dict, list)): + values.append(json.dumps(value)) + else: + values.append(value) + + colNames = ", ".join([f'"{col}"' for col in columns]) + placeholders = ", ".join(["%s"] * len(columns)) + + updateCols = [col for col in columns if col not in ["id", "_createdAt", "_createdBy"]] + updateClause = ", ".join([f'"{col}" = EXCLUDED."{col}"' for col in updateCols]) + + if updateClause: + sql = f''' + INSERT INTO "{tableName}" ({colNames}) + VALUES ({placeholders}) + ON CONFLICT ("id") DO UPDATE SET {updateClause} + ''' + else: + sql = f''' + INSERT INTO "{tableName}" ({colNames}) + VALUES ({placeholders}) + ON CONFLICT ("id") DO NOTHING + ''' + + try: + with conn.cursor() as cursor: + cursor.execute(sql, values) + return True + except Exception as e: + logger.error(f"Fehler beim Einfügen in {tableName}: {e}") + return False + + +def loadMigrationFile(filePath: str) -> Dict[str, Any]: + """Lädt die Migrationsdatei.""" + logger.info(f"Lade Migrationsdatei: {filePath}") + + if not os.path.exists(filePath): + raise FileNotFoundError(f"Datei nicht gefunden: {filePath}") + + with open(filePath, "r", encoding="utf-8") as f: + data = json.load(f) + + # Validierung - unterstütze beide Formate (alt: tables, neu: databases) + if "databases" not in data and "tables" not in data: + raise ValueError("Ungültiges Migrationsformat: 'databases' oder 'tables' erforderlich") + + return data + + +def _importSingleDatabase( + dbName: str, + dbData: Dict[str, Any], + dryRun: bool, + clearFirst: bool, + onlyTables: Optional[List[str]] +) -> Dict[str, Any]: + """Importiert Daten in eine einzelne Datenbank.""" + stats = { + "imported": {}, + "skipped": {}, + "errors": {}, + "totalImported": 0, + "totalSkipped": 0, + "totalErrors": 0 + } + + conn = _getDbConnection(dbName) + if conn is None: + logger.warning(f" Datenbank '{dbName}' existiert nicht - übersprungen") + return stats + + try: + existingTables = _getExistingTables(conn) + tables = dbData.get("tables", {}) + + tablesToImport = list(tables.keys()) + if onlyTables: + tablesToImport = [t for t in tablesToImport if t in onlyTables] + + for tableName in tablesToImport: + records = tables[tableName] + + if tableName not in existingTables: + logger.warning(f" Tabelle '{tableName}' existiert nicht - übersprungen") + stats["skipped"][tableName] = len(records) + stats["totalSkipped"] += len(records) + continue + + if dryRun: + stats["imported"][tableName] = len(records) + stats["totalImported"] += len(records) + continue + + if clearFirst: + _clearTable(conn, tableName) + + existingColumns = _getTableColumns(conn, tableName) + + imported = 0 + errors = 0 + + for record in records: + if _insertRecord(conn, tableName, record, existingColumns): + imported += 1 + else: + errors += 1 + + stats["imported"][tableName] = imported + stats["totalImported"] += imported + + if errors > 0: + stats["errors"][tableName] = errors + stats["totalErrors"] += errors + + if imported > 0: + logger.info(f" {tableName}: {imported} importiert, {errors} Fehler") + + if not dryRun: + conn.commit() + else: + conn.rollback() + + return stats + + except Exception as e: + conn.rollback() + logger.error(f" Import fehlgeschlagen: {e}") + raise + + finally: + conn.close() + + +def importDatabase( + filePath: str, + dryRun: bool = False, + clearFirst: bool = False, + onlyTables: Optional[List[str]] = None, + onlyDatabases: Optional[List[str]] = None +) -> Dict[str, Any]: + """ + Importiert Daten aus einer Migrationsdatei. + + Args: + filePath: Pfad zur Migrationsdatei + dryRun: Nur simulieren + clearFirst: Tabellen vor Import leeren + onlyTables: Nur diese Tabellen importieren + onlyDatabases: Nur diese Datenbanken importieren + + Returns: + Import-Statistiken + """ + migrationData = loadMigrationFile(filePath) + meta = migrationData.get("meta", {}) + + logger.info(f"Migrationsdatei geladen:") + logger.info(f" Exportiert am: {meta.get('exportedAt', 'unbekannt')}") + logger.info(f" Quelle: {meta.get('exportedFrom', 'unbekannt')}") + + stats = { + "databases": {}, + "totalImported": 0, + "totalSkipped": 0, + "totalErrors": 0 + } + + # Neues Format (mehrere Datenbanken) + if "databases" in migrationData: + databases = migrationData["databases"] + logger.info(f" Datenbanken: {len(databases)}") + logger.info(f" Tabellen: {meta.get('totalTables', 'unbekannt')}") + logger.info(f" Datensätze: {meta.get('totalRecords', 'unbekannt')}") + + for dbName, dbData in databases.items(): + if onlyDatabases and dbName not in onlyDatabases: + continue + + logger.info(f"Importiere Datenbank: {dbName}") + dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) + + stats["databases"][dbName] = dbStats + stats["totalImported"] += dbStats["totalImported"] + stats["totalSkipped"] += dbStats["totalSkipped"] + stats["totalErrors"] += dbStats["totalErrors"] + + # Altes Format (einzelne Datenbank - poweron_app) + elif "tables" in migrationData: + logger.info(" Format: Legacy (einzelne Datenbank)") + dbName = "poweron_app" + dbData = {"tables": migrationData["tables"]} + + if not onlyDatabases or dbName in onlyDatabases: + logger.info(f"Importiere Datenbank: {dbName}") + dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) + + stats["databases"][dbName] = dbStats + stats["totalImported"] = dbStats["totalImported"] + stats["totalSkipped"] = dbStats["totalSkipped"] + stats["totalErrors"] = dbStats["totalErrors"] + + if dryRun: + logger.info("Dry-Run: Keine Änderungen vorgenommen") + + return stats + + +def printImportPreview(filePath: str): + """Zeigt eine Vorschau der zu importierenden Daten.""" + migrationData = loadMigrationFile(filePath) + meta = migrationData.get("meta", {}) + + print("\n" + "=" * 70) + print("IMPORT VORSCHAU") + print("=" * 70) + print(f"Datei: {filePath}") + print(f"Exportiert am: {meta.get('exportedAt', 'unbekannt')}") + print(f"Quelle: {meta.get('exportedFrom', 'unbekannt')}") + + # Neues Format + if "databases" in migrationData: + databases = migrationData["databases"] + print(f"Datenbanken: {len(databases)}") + print("=" * 70) + + grandTotal = 0 + for dbName, dbData in databases.items(): + tables = dbData.get("tables", {}) + dbTotal = sum(len(records) for records in tables.values()) + grandTotal += dbTotal + + print(f"\n{dbName} ({dbTotal} Datensätze)") + print("-" * 70) + print(f" {'Tabelle':<45} {'Datensätze':>15}") + print(f" {'-' * 45} {'-' * 15}") + + for tableName, records in sorted(tables.items()): + if len(records) > 0: + print(f" {tableName:<45} {len(records):>15}") + + print("\n" + "=" * 70) + print(f"GESAMT: {grandTotal} Datensätze") + + # Altes Format + elif "tables" in migrationData: + tables = migrationData["tables"] + print(f"Format: Legacy (poweron_app)") + print("-" * 70) + print(f"{'Tabelle':<45} {'Datensätze':>15}") + print("-" * 70) + + totalRecords = 0 + for tableName, records in sorted(tables.items()): + count = len(records) + totalRecords += count + if count > 0: + print(f"{tableName:<45} {count:>15}") + + print("-" * 70) + print(f"{'GESAMT':<45} {totalRecords:>15}") + + print("=" * 70 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Importiert Datenbank-Daten aus einer Migrationsdatei", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Datenbanken: + poweron_app - User, Mandate, RBAC, Features + poweron_chat - Chat-Konversationen + poweron_management - Workflows, Prompts, Connections + poweron_realestate - Real Estate Daten + poweron_trustee - Trustee Daten + +Beispiele: + python tool_db_import_migration.py migration_export.json --dry-run + python tool_db_import_migration.py migration_export.json --preview + python tool_db_import_migration.py migration_export.json --force + python tool_db_import_migration.py migration_export.json --clear-first --force + python tool_db_import_migration.py migration_export.json --only-db poweron_app + python tool_db_import_migration.py migration_export.json --only-tables UserInDB,Mandate + """ + ) + + parser.add_argument( + "import_file", + help="Pfad zur Migrationsdatei (JSON)", + type=str + ) + + parser.add_argument( + "--dry-run", + help="Simuliert den Import ohne Änderungen", + action="store_true" + ) + + parser.add_argument( + "--force", + help="Bestätigung überspringen", + action="store_true" + ) + + parser.add_argument( + "--clear-first", + help="Tabellen vor dem Import leeren", + action="store_true" + ) + + parser.add_argument( + "--only-tables", + help="Nur diese Tabellen importieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--only-db", + help="Nur diese Datenbank(en) importieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--preview", + help="Nur Vorschau anzeigen (kein Import)", + action="store_true" + ) + + args = parser.parse_args() + + # Nur Vorschau anzeigen + if args.preview: + printImportPreview(args.import_file) + return + + # Listen parsen + onlyTables = None + if args.only_tables: + onlyTables = [t.strip() for t in args.only_tables.split(",") if t.strip()] + + onlyDatabases = None + if args.only_db: + onlyDatabases = [db.strip() for db in args.only_db.split(",") if db.strip()] + + # Bestätigung einholen + if not args.dry_run and not args.force: + printImportPreview(args.import_file) + + if args.clear_first: + print("WARNUNG: --clear-first wird ALLE bestehenden Daten in den Zieltabellen löschen!") + + response = input("\nMöchten Sie den Import starten? [y/N]: ") + if response.lower() not in ["y", "yes", "j", "ja"]: + print("Import abgebrochen.") + return + + # Import durchführen + try: + if args.dry_run: + logger.info("=== DRY-RUN MODUS ===") + + stats = importDatabase( + filePath=args.import_file, + dryRun=args.dry_run, + clearFirst=args.clear_first, + onlyTables=onlyTables, + onlyDatabases=onlyDatabases + ) + + print("\n" + "=" * 70) + print("IMPORT ERGEBNIS") + print("=" * 70) + print(f"Importiert: {stats['totalImported']} Datensätze") + print(f"Übersprungen: {stats['totalSkipped']} Datensätze") + print(f"Fehler: {stats['totalErrors']} Datensätze") + + if args.dry_run: + print("\n(Dry-Run: Keine tatsächlichen Änderungen vorgenommen)") + else: + print("\n Import erfolgreich abgeschlossen!") + + print("=" * 70 + "\n") + + except Exception as e: + logger.error(f"Import fehlgeschlagen: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 7a9b264170baa29cafcbc593c73c31838f53dc24 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 19 Jan 2026 16:25:04 +0100 Subject: [PATCH 03/32] fixed transactions --- .../migration_export_20260119_085558.json | 1221 ----------------- tool_db_export_migration.py | 268 +++- 2 files changed, 264 insertions(+), 1225 deletions(-) delete mode 100644 local/backup/migration_export_20260119_085558.json diff --git a/local/backup/migration_export_20260119_085558.json b/local/backup/migration_export_20260119_085558.json deleted file mode 100644 index 50cc9e5a..00000000 --- a/local/backup/migration_export_20260119_085558.json +++ /dev/null @@ -1,1221 +0,0 @@ -{ - "meta": { - "exportedAt": "2026-01-19T07:55:59.185004Z", - "exportedFrom": "Development Instance Patrick", - "databaseName": "poweron_app", - "version": "1.0", - "tableCount": 6, - "excludedTables": [ - "_system" - ], - "includesMeta": false, - "totalRecords": 107 - }, - "tables": { - "AccessRule": [ - { - "id": "90990e75-ac45-4986-9c09-f299f198d2b3", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": null, - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "a9536849-a53b-458d-bab8-77a2a3ac2747", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": null, - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "n" - }, - { - "id": "54870935-d5a1-41b7-b088-30f70b4dc835", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": null, - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "a3743435-1015-4bd1-a256-4f91eda9f544", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": null, - "view": 1, - "read": "g", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "3e7b79b5-a68a-41b5-9e7f-f8159a310ed3", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Mandate", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "e7e8818c-04ed-473a-b1da-c7095f98f99e", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Mandate", - "view": 0, - "read": "n", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "61714126-2822-48cd-bbb5-175c141b2c0b", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Mandate", - "view": 0, - "read": "n", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "0ae9dcb8-302a-44cb-b777-1f9d26af7190", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Mandate", - "view": 0, - "read": "n", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "2ab27800-df9d-4307-aee3-d99064fbab5d", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "UserInDB", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "279c6538-401a-4cd3-a36a-d1399f8bd2e4", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "UserInDB", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "4ab1abbe-abf1-4510-8ba1-4afbb37b1fd1", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "UserInDB", - "view": 1, - "read": "m", - "create": "n", - "update": "m", - "delete": "n" - }, - { - "id": "86c384a7-a2b5-4207-9e62-42cc50a2d20b", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "UserInDB", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "542af362-50c8-4e19-af21-5db2e0b8733e", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "UserConnection", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "8e356f1c-134d-4115-962e-0d0b7b71cf65", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "UserConnection", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "061b7a09-1003-4250-9524-64d648135467", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "UserConnection", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "89704bf9-d742-4149-a1e2-dcaaba9957c9", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "UserConnection", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "d296b5fb-48c3-4351-aa84-70ec1593369e", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "DataNeutraliserConfig", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "6ac892f8-69b6-4c2f-b18c-f5d58f8c95d9", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "DataNeutraliserConfig", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "defc2070-098f-44e2-9c59-264189730cc7", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "DataNeutraliserConfig", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "5927b71b-51f7-4671-a91b-f4f300af04f7", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "DataNeutraliserConfig", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "5a1907e7-6772-4b3f-9d34-6b847db3c14c", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "DataNeutralizerAttributes", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "c4eac0f8-e9d2-4064-80bd-a4e9c9e68686", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "DataNeutralizerAttributes", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "77a76ccf-91bb-4348-9caf-912b2bd7fe02", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "DataNeutralizerAttributes", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "d059cf52-8c56-4273-80f9-e7b8b40ebb4b", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "DataNeutralizerAttributes", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "0ca0ed8b-9327-43ff-b7b5-dc5870fb09c2", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "ChatWorkflow", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "a72168c5-bf33-43ef-a708-fd40e8feb6ba", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "ChatWorkflow", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "9c80b57b-d2ed-4309-9e87-62a98fe144d0", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "ChatWorkflow", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "4e534940-33cc-43ac-b859-9360a2e60cf1", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "ChatWorkflow", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "b0b3e37e-ea48-471e-95f6-b8e1e62ad09b", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Prompt", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "0f0f4c89-2aa5-4755-aeea-562bd020593d", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Prompt", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "0185dcc1-3560-4255-9840-8ab5db8042f8", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Prompt", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "c9c87948-21ce-47f4-8040-c02a0917aeb2", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Prompt", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "9e5f6dd5-c7a7-4a22-a1d6-82d4c68630a0", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Projekt", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "852f4f77-dac7-46de-8136-ebc46344101e", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Projekt", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "61710630-3c71-4adf-a997-b9e61d06fc00", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Projekt", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "158789bd-7f26-4d88-a27e-6ce11b3d94f9", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Projekt", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "c96b8b90-df7b-49ea-8ec9-0754c6179561", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Parzelle", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "cdb610f9-21c6-4e8d-b9b6-d4a44f372ad6", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Parzelle", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "6d3ff830-ae8f-4447-89a8-eaa8f8c4ecc7", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Parzelle", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "7791781c-810a-43ac-a5ce-d84a3584e5f4", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Parzelle", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "f49cd1cb-bdcd-4a9a-92fb-205078a3acba", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Dokument", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "44d5455e-a17b-4b6d-a426-21565fe1cef7", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Dokument", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "b2b58648-a555-4135-a038-84218a279437", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Dokument", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "073195c6-01bc-40f9-9896-f3c10ebce288", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Dokument", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "ba537ef0-239b-456d-b354-dec2c9d921ed", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Gemeinde", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "3d66cd04-f4e6-43f8-ab77-c31a87730096", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Gemeinde", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "bcc52c15-b41c-4a97-8e13-10035dd22202", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Gemeinde", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "844c7c02-dcc8-43a4-8eb5-f8ec198f50fd", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Gemeinde", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "15c1c76d-b8f7-40d2-8027-5ffdf6c96e28", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Kanton", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "b73379ef-3dbf-4118-b6f2-473f3c1c52e7", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Kanton", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "afc14c03-e953-4186-a923-4a12d3c6a312", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Kanton", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "c49ee8f3-f295-465d-be92-9bf46ee259e0", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Kanton", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "fcadb856-46f4-4ff0-85eb-67df3891c25d", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "Land", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "4a4e52ad-4ee3-4044-9e14-e0d797356139", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "Land", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "297ae474-ee2d-42f0-af09-eaa8b48d9684", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "Land", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "a567d373-00be-4c7e-b25e-deb6b10b57a3", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "Land", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "8f961779-1790-4cad-bebc-9b9bc436a29c", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteeOrganisation", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "81be065a-9ec6-4713-9fca-f5087cb9c3ee", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteeOrganisation", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "1fe2a920-0f9d-4c33-9288-4381af8c523e", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteeOrganisation", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "e99b39eb-ed55-4d08-a5d5-c3cf4645c312", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteeOrganisation", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "a54eed2a-e269-4884-ac2f-c09990f9b3d2", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteeRole", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "2bd073ca-30c3-43e0-8ea1-d77241053e7c", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteeRole", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "557ba1d5-2be0-43ee-a842-d0e74b45df41", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteeRole", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "81b46bcc-6cc3-40e0-9d82-f3a5bcc23b53", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteeRole", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "f3a58145-8986-42a7-bce4-eab0a1fa0962", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteeAccess", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "31d2a772-ed6b-447b-aa95-16a160d39601", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteeAccess", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "8a373342-8b8b-4a64-afd4-9f3ae8071d28", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteeAccess", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "b7a501e1-793f-47b6-b20c-1bfa61531cee", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteeAccess", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "0e67e101-5ce6-40fb-a783-104eebe29f92", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteeContract", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "23005b91-dc46-4bbe-a690-6c4568484858", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteeContract", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "aec6753b-cd7d-4ec8-8fa6-b9d007eb3657", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteeContract", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "31bf3c95-1012-4d61-91fc-36e4e3b832ec", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteeContract", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "17edb067-e62e-489a-842b-adbe4588aec0", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteeDocument", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "981e57b7-d813-4d67-9cf9-73576cc2affb", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteeDocument", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "3078980e-e8f2-44f0-b172-4a4c877b1b14", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteeDocument", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "f25efabc-e861-4cba-ad9c-6c8886f99ae5", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteeDocument", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "178c1dc4-7879-4a98-bbdd-54c1c55500c2", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteePosition", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "bbf02493-5ba2-4c0d-83b7-2a1c20e41108", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteePosition", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "7183b536-9a8a-4c2e-b6f4-4f69cc8a50b4", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteePosition", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "309130ca-254d-45b0-8629-47fe6a391a34", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteePosition", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "240fdaa4-75df-47a4-b33e-bce2e09dc741", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "TrusteePositionDocument", - "view": 1, - "read": "a", - "create": "a", - "update": "a", - "delete": "a" - }, - { - "id": "528ba11a-824d-477a-ae2f-aa453fdea2f2", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "TrusteePositionDocument", - "view": 1, - "read": "g", - "create": "g", - "update": "g", - "delete": "g" - }, - { - "id": "e3efeb99-eeb2-4450-85a4-f4261449a9ea", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "TrusteePositionDocument", - "view": 1, - "read": "m", - "create": "m", - "update": "m", - "delete": "m" - }, - { - "id": "ca4bf93c-311c-4d2b-8b6e-9bd879ff5cde", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "TrusteePositionDocument", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "186bc21d-eb17-4d4a-ae4e-20725768befb", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "DATA", - "item": "AuthEvent", - "view": 1, - "read": "a", - "create": "n", - "update": "n", - "delete": "a" - }, - { - "id": "73d6eaa4-c1d1-4bfb-a2d6-a1772f9fd31c", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "DATA", - "item": "AuthEvent", - "view": 1, - "read": "a", - "create": "n", - "update": "n", - "delete": "a" - }, - { - "id": "2dce96a0-9a30-4d83-8544-29b60b7e8199", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "DATA", - "item": "AuthEvent", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "91b2e3ef-7472-41b0-81ef-7e891ce7d183", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "DATA", - "item": "AuthEvent", - "view": 1, - "read": "m", - "create": "n", - "update": "n", - "delete": "n" - }, - { - "id": "5bf4d7af-d8e9-4ca0-a9d1-c4cdd6a8b2a8", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "UI", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "37999b49-c13c-44a1-a94e-41477d2e2e29", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "UI", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "b7af378c-0da9-4554-ab9e-3e4d5130778b", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "UI", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "d0c5bc55-d6e7-40c2-8bab-b7df13836712", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "UI", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "96827d13-bb10-4341-9615-03b940e4eab1", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "context": "RESOURCE", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "c052f46c-07b7-447b-885d-15803f615b6a", - "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "context": "RESOURCE", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "91213ddd-9490-4f9f-928a-5f7c87bb6e05", - "roleId": "bc22885c-5354-463e-a3fe-480941e016df", - "context": "RESOURCE", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - }, - { - "id": "58abcdc0-c8ba-435c-b01e-7b9c52581251", - "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "context": "RESOURCE", - "item": null, - "view": 1, - "read": null, - "create": null, - "update": null, - "delete": null - } - ], - "Mandate": [ - { - "id": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", - "name": "Root", - "enabled": 1 - } - ], - "Role": [ - { - "id": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", - "roleLabel": "sysadmin", - "description": { - "en": "System Administrator - Full access to all system resources", - "fr": "Administrateur système - Accès complet à toutes les ressources", - "ge": null, - "it": null - }, - "mandateId": null, - "featureInstanceId": null, - "featureCode": null, - "isSystemRole": 1 - }, - { - "id": "9d7af325-2dc9-451f-88d1-090dc06de3db", - "roleLabel": "admin", - "description": { - "en": "Administrator - Manage users and resources within mandate scope", - "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat", - "ge": null, - "it": null - }, - "mandateId": null, - "featureInstanceId": null, - "featureCode": null, - "isSystemRole": 1 - }, - { - "id": "bc22885c-5354-463e-a3fe-480941e016df", - "roleLabel": "user", - "description": { - "en": "User - Standard user with access to own records", - "fr": "Utilisateur - Utilisateur standard avec accès à ses propres enregistrements", - "ge": null, - "it": null - }, - "mandateId": null, - "featureInstanceId": null, - "featureCode": null, - "isSystemRole": 1 - }, - { - "id": "95a88cf7-8a2a-42b2-b136-168966ad86b5", - "roleLabel": "viewer", - "description": { - "en": "Viewer - Read-only access to group records", - "fr": "Visualiseur - Accès en lecture seule aux enregistrements du groupe", - "ge": null, - "it": null - }, - "mandateId": null, - "featureInstanceId": null, - "featureCode": null, - "isSystemRole": 1 - } - ], - "UserInDB": [ - { - "id": "3876d530-29d8-451d-a2fc-92af5cb2b817", - "username": "admin", - "email": "admin@example.com", - "fullName": "Administrator", - "language": "en", - "enabled": 1, - "isSysAdmin": 1, - "roleLabels": [ - "sysadmin" - ], - "authenticationAuthority": "local", - "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", - "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$Sumds1bqvZfyPseYs/YeYw$eiRcnO7J+ebit2oV6Ndqaer2ZIgPErTC9q2riRpiiwA", - "resetToken": null, - "resetTokenExpires": null - }, - { - "id": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", - "username": "event", - "email": "event@example.com", - "fullName": "Event", - "language": "en", - "enabled": 1, - "isSysAdmin": 1, - "roleLabels": [ - "sysadmin" - ], - "authenticationAuthority": "local", - "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", - "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$BoDwnlNq7d3b2ztnrPWecw$ICkAaTjE/R39CJ7MryLmfmeEX5m4N/6S3HaDfOZuOBM", - "resetToken": null, - "resetTokenExpires": null - } - ], - "UserMandate": [ - { - "id": "70f9e733-7297-4afe-8784-9f7c730de3c4", - "userId": "3876d530-29d8-451d-a2fc-92af5cb2b817", - "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", - "enabled": 1 - }, - { - "id": "412ba93d-2abe-4916-8a80-bec4672c0baf", - "userId": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", - "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", - "enabled": 1 - } - ], - "UserMandateRole": [ - { - "id": "cff2dd70-4dd6-4028-a384-7b0e8578f9dc", - "userMandateId": "70f9e733-7297-4afe-8784-9f7c730de3c4", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" - }, - { - "id": "54232df1-b559-44bb-8a51-adbd82818d94", - "userMandateId": "412ba93d-2abe-4916-8a80-bec4672c0baf", - "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" - } - ] - }, - "summary": { - "AccessRule": { - "recordCount": 96 - }, - "Mandate": { - "recordCount": 1 - }, - "Role": { - "recordCount": 4 - }, - "UserInDB": { - "recordCount": 2 - }, - "UserMandate": { - "recordCount": 2 - }, - "UserMandateRole": { - "recordCount": 2 - } - } -} \ No newline at end of file diff --git a/tool_db_export_migration.py b/tool_db_export_migration.py index 09aa2f8f..6c93370a 100644 --- a/tool_db_export_migration.py +++ b/tool_db_export_migration.py @@ -5,6 +5,8 @@ Datenbank Export-Tool für Migration. Dieses Script exportiert alle Daten aus ALLEN PowerOn PostgreSQL-Datenbanken in eine JSON-Datei, die als Migrationsdatensatz verwendet werden kann. +Zusätzlich wird eine separate JSON-Datei mit nur den Strukturen (ohne Daten) +erstellt: _structure.json Datenbanken: - poweron_app (User, Mandate, RBAC, Features, etc.) @@ -18,6 +20,7 @@ Verwendung: Optionen: --output, -o Pfad zur Ausgabedatei (Standard: migration_export_.json) + Die Struktur-Datei wird automatisch als _structure.json erstellt --pretty, -p JSON formatiert ausgeben (für bessere Lesbarkeit) --exclude Komma-getrennte Liste von Tabellen, die ausgeschlossen werden sollen --include-meta System-Metadaten (_createdAt, _modifiedAt, etc.) beibehalten @@ -150,6 +153,8 @@ def _getDbConnection(dbDatabase: str): password=dbPassword, cursor_factory=psycopg2.extras.RealDictCursor ) + # Autocommit muss VOR set_client_encoding gesetzt werden, um Transaction-Konflikte zu vermeiden + conn.autocommit = True conn.set_client_encoding('UTF8') return conn except Exception as e: @@ -205,6 +210,222 @@ def _getTableRowCount(conn, tableName: str) -> int: return result["count"] if result else 0 +def _getTableStructure(conn, tableName: str) -> Dict[str, Any]: + """Holt die Struktur einer Tabelle (Spalten, Constraints, Indizes) ohne Daten.""" + structure = { + "columns": [], + "primaryKeys": [], + "foreignKeys": [], + "uniqueConstraints": [], + "indexes": [], + "checkConstraints": [] + } + + # Connection hat bereits autocommit = True, daher keine Transaction-Probleme + with conn.cursor() as cursor: + # Spalten-Informationen + cursor.execute(""" + SELECT + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale, + is_nullable, + column_default, + udt_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = %s + ORDER BY ordinal_position + """, (tableName,)) + + for row in cursor.fetchall(): + colInfo = { + "name": row["column_name"], + "type": row["data_type"], + "udtName": row["udt_name"], + "nullable": row["is_nullable"] == "YES", + "default": row["column_default"] + } + + if row["character_maximum_length"]: + colInfo["maxLength"] = row["character_maximum_length"] + if row["numeric_precision"]: + colInfo["precision"] = row["numeric_precision"] + if row["numeric_scale"]: + colInfo["scale"] = row["numeric_scale"] + + structure["columns"].append(colInfo) + + # Primary Keys + cursor.execute(""" + SELECT + kcu.column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + WHERE tc.table_schema = 'public' + AND tc.table_name = %s + AND tc.constraint_type = 'PRIMARY KEY' + ORDER BY kcu.ordinal_position + """, (tableName,)) + + structure["primaryKeys"] = [row["column_name"] for row in cursor.fetchall()] + + # Foreign Keys + cursor.execute(""" + SELECT + kcu.column_name, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name, + tc.constraint_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = 'public' + AND tc.table_name = %s + """, (tableName,)) + + for row in cursor.fetchall(): + structure["foreignKeys"].append({ + "column": row["column_name"], + "referencesTable": row["foreign_table_name"], + "referencesColumn": row["foreign_column_name"], + "constraintName": row["constraint_name"] + }) + + # Unique Constraints - FIX: Tabellen-Aliase verwenden um ambiguous columns zu vermeiden + cursor.execute(""" + SELECT + kcu.column_name, + tc.constraint_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + AND tc.table_name = kcu.table_name + WHERE tc.table_schema = 'public' + AND tc.table_name = %s + AND tc.constraint_type = 'UNIQUE' + ORDER BY kcu.ordinal_position + """, (tableName,)) + + uniqueGroups = {} + for row in cursor.fetchall(): + constraintName = row["constraint_name"] + if constraintName not in uniqueGroups: + uniqueGroups[constraintName] = [] + uniqueGroups[constraintName].append(row["column_name"]) + + structure["uniqueConstraints"] = [ + {"columns": cols, "constraintName": name} + for name, cols in uniqueGroups.items() + ] + + # Indizes (ohne Primary Key und Unique Constraints) + cursor.execute(""" + SELECT + i.relname AS index_name, + a.attname AS column_name, + ix.indisunique AS is_unique + FROM pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) + WHERE t.relkind = 'r' + AND t.relname = %s + AND NOT ix.indisprimary + ORDER BY i.relname, a.attnum + """, (tableName,)) + + indexGroups = {} + for row in cursor.fetchall(): + indexName = row["index_name"] + if indexName not in indexGroups: + indexGroups[indexName] = { + "name": indexName, + "columns": [], + "unique": row["is_unique"] + } + indexGroups[indexName]["columns"].append(row["column_name"]) + + structure["indexes"] = list(indexGroups.values()) + + # Check Constraints - FIX: Tabellen-Aliase verwenden + cursor.execute(""" + SELECT + cc.constraint_name, + cc.check_clause + FROM information_schema.check_constraints cc + JOIN information_schema.constraint_column_usage ccu + ON cc.constraint_name = ccu.constraint_name + WHERE ccu.table_schema = 'public' + AND ccu.table_name = %s + """, (tableName,)) + + for row in cursor.fetchall(): + structure["checkConstraints"].append({ + "constraintName": row["constraint_name"], + "checkClause": row["check_clause"] + }) + + return structure + + +def _exportSingleDatabaseStructure( + dbDatabase: str, + excludeTables: List[str] +) -> Optional[Dict[str, Any]]: + """Exportiert nur die Struktur einer einzelnen Datenbank (ohne Daten).""" + conn = _getDbConnection(dbDatabase) + + if conn is None: + return None + + try: + allTables = _getTables(conn) + + # System-Tabellen ausschliessen + systemTables = ["_system"] + tablesToExport = [ + t for t in allTables + if t not in systemTables and t not in excludeTables + ] + + dbExport = { + "tables": {}, + "summary": {}, + "tableCount": len(tablesToExport) + } + + for tableName in tablesToExport: + try: + structure = _getTableStructure(conn, tableName) + dbExport["tables"][tableName] = structure + dbExport["summary"][tableName] = { + "columnCount": len(structure["columns"]), + "primaryKeyCount": len(structure["primaryKeys"]), + "foreignKeyCount": len(structure["foreignKeys"]), + "indexCount": len(structure["indexes"]) + } + + logger.info(f" {tableName}: {len(structure['columns'])} Spalten") + + except Exception as e: + logger.error(f" Fehler bei Tabelle {tableName}: {e}") + dbExport["tables"][tableName] = {} + dbExport["summary"][tableName] = {"error": str(e)} + + return dbExport + + finally: + conn.close() + + def _exportSingleDatabase( dbDatabase: str, excludeTables: List[str], @@ -249,6 +470,7 @@ def _exportSingleDatabase( logger.error(f" Fehler bei Tabelle {tableName}: {e}") dbExport["tables"][tableName] = [] dbExport["summary"][tableName] = {"recordCount": 0, "error": str(e)} + # Bei autocommit = True ist kein rollback() notwendig return dbExport @@ -265,6 +487,7 @@ def exportDatabase( ) -> str: """ Exportiert alle Datenbanken in eine JSON-Datei. + Erstellt zusätzlich eine separate JSON-Datei mit nur den Strukturen (ohne Daten). Args: outputPath: Pfad zur Ausgabedatei (optional) @@ -310,12 +533,31 @@ def exportDatabase( "databases": {} } + # Struktur-Export erstellen + structureData = { + "meta": { + "exportedAt": datetime.utcnow().isoformat() + "Z", + "exportedFrom": _getConfigValue("APP_ENV_LABEL", "unknown"), + "version": "1.0", + "databaseCount": 0, + "totalTables": 0, + "excludedTables": excludeTables, + "note": "Nur Strukturen, keine Daten" + }, + "databases": {} + } + # Jede Datenbank exportieren for dbName in databasesToExport: logger.info(f"Exportiere Datenbank: {dbName}") + # Daten exportieren dbExport = _exportSingleDatabase(dbName, excludeTables, includeMeta) + # Struktur exportieren + logger.info(f"Exportiere Struktur für Datenbank: {dbName}") + dbStructure = _exportSingleDatabaseStructure(dbName, excludeTables) + if dbExport is not None: exportData["databases"][dbName] = dbExport exportData["meta"]["databaseCount"] += 1 @@ -324,8 +566,13 @@ def exportDatabase( logger.info(f" -> {dbExport['tableCount']} Tabellen, {dbExport['totalRecords']} Datensätze") else: logger.info(f" -> Übersprungen (existiert nicht)") + + if dbStructure is not None: + structureData["databases"][dbName] = dbStructure + structureData["meta"]["databaseCount"] += 1 + structureData["meta"]["totalTables"] += dbStructure["tableCount"] - # JSON-Datei schreiben + # JSON-Datei mit Daten schreiben logger.info(f"Schreibe Exportdatei: {outputPath}") with open(outputPath, "w", encoding="utf-8") as f: @@ -334,16 +581,29 @@ def exportDatabase( else: json.dump(exportData, f, ensure_ascii=False, default=str) - # Dateigrösse berechnen + # JSON-Datei mit Strukturen schreiben + structurePath = outputPath.replace(".json", "_structure.json") + logger.info(f"Schreibe Struktur-Exportdatei: {structurePath}") + + with open(structurePath, "w", encoding="utf-8") as f: + if prettyPrint: + json.dump(structureData, f, indent=2, ensure_ascii=False, default=str) + else: + json.dump(structureData, f, ensure_ascii=False, default=str) + + # Dateigrössen berechnen fileSize = os.path.getsize(outputPath) fileSizeStr = _formatFileSize(fileSize) + structureFileSize = os.path.getsize(structurePath) + structureFileSizeStr = _formatFileSize(structureFileSize) + logger.info(f"Export abgeschlossen!") logger.info(f" Datenbanken: {exportData['meta']['databaseCount']}") logger.info(f" Tabellen: {exportData['meta']['totalTables']}") logger.info(f" Datensätze: {exportData['meta']['totalRecords']}") - logger.info(f" Dateigrösse: {fileSizeStr}") - logger.info(f" Ausgabedatei: {outputPath}") + logger.info(f" Daten-Export: {fileSizeStr} - {outputPath}") + logger.info(f" Struktur-Export: {structureFileSizeStr} - {structurePath}") return outputPath From 77e14147441d69bcf5a0a3bce88559f328fa8ccf Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 20 Jan 2026 00:55:39 +0100 Subject: [PATCH 04/32] module testing --- .../{datamodelChatbot.py => datamodelChat.py} | 0 modules/datamodels/datamodelWorkflow.py | 2 +- .../datamodels/datamodelWorkflowActions.py | 2 +- modules/features/chatbot/mainChatbot.py | 4 +- modules/features/workflow/mainWorkflow.py | 2 +- modules/interfaces/interfaceDbChatbot.py | 2 +- modules/routes/routeAdminRbacRoles.py | 11 +- modules/routes/routeDataAutomation.py | 2 +- modules/routes/routeDataMandates.py | 84 +-- modules/routes/routeDataUsers.py | 70 +- modules/routes/routeFeatureChatDynamic.py | 2 +- modules/routes/routeFeatureChatbot.py | 2 +- modules/routes/routeFeatures.py | 255 ++++++-- modules/routes/routeWorkflows.py | 2 +- modules/services/__init__.py | 2 +- modules/services/serviceAi/mainServiceAi.py | 2 +- .../serviceAi/subContentExtraction.py | 2 +- .../services/serviceAi/subDocumentIntents.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../mainServiceExtraction.py | 2 +- .../mainServiceGeneration.py | 2 +- modules/shared/dbMultiTenantOptimizations.py | 110 +++- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 4 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 12 +- modules/workflows/workflowManager.py | 10 +- tests/functional/test02_ai_models.py | 2 +- tests/functional/test03_ai_operations.py | 6 +- tests/functional/test04_ai_behavior.py | 2 +- .../test05_workflow_with_documents.py | 2 +- .../test06_workflow_prompt_variations.py | 2 +- .../test09_document_generation_formats.py | 2 +- .../test10_document_generation_formats.py | 2 +- .../test11_code_generation_formats.py | 2 +- .../workflows/test_workflow_execution.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- tool_db_adapt_to_models.py | 428 ++++++++++++ tool_db_export_migration.py | 188 ++++-- tool_db_import_migration.py | 612 ------------------ 81 files changed, 1022 insertions(+), 922 deletions(-) rename modules/datamodels/{datamodelChatbot.py => datamodelChat.py} (100%) create mode 100644 tool_db_adapt_to_models.py delete mode 100644 tool_db_import_migration.py diff --git a/modules/datamodels/datamodelChatbot.py b/modules/datamodels/datamodelChat.py similarity index 100% rename from modules/datamodels/datamodelChatbot.py rename to modules/datamodels/datamodelChat.py diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py index 19117fce..b884382c 100644 --- a/modules/datamodels/datamodelWorkflow.py +++ b/modules/datamodels/datamodelWorkflow.py @@ -14,7 +14,7 @@ from modules.datamodels.datamodelDocref import DocumentReferenceList # Forward references for circular imports (use string annotations) if TYPE_CHECKING: - from modules.datamodels.datamodelChatbot import ChatDocument, ActionResult + from modules.datamodels.datamodelChat import ChatDocument, ActionResult from modules.datamodels.datamodelExtraction import ExtractionOptions diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index 1ca90d51..8bac1fd5 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index a5222966..43503339 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -13,7 +13,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -335,7 +335,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChatbot import ChatLog + from modules.datamodels.datamodelChat import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/features/workflow/mainWorkflow.py b/modules/features/workflow/mainWorkflow.py index ab92510c..70a2e9aa 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/features/workflow/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/interfaces/interfaceDbChatbot.py b/modules/interfaces/interfaceDbChatbot.py index 9ded2dd8..c9f87a55 100644 --- a/modules/interfaces/interfaceDbChatbot.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( ChatDocument, ChatStat, ChatLog, diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index caa39859..a9397867 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -363,7 +363,16 @@ async def listUsersWithRoles( interface = getRootInterface() # Get all users (SysAdmin sees all) - users = interface.getUsers() + # Use db.getRecordset with UserInDB (the actual database model) + from modules.datamodels.datamodelUam import User, UserInDB + allUsersData = interface.db.getRecordset(UserInDB) + # Convert to User objects, filtering out sensitive fields + users = [] + for u in allUsersData: + cleanedUser = {k: v for k, v in u.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + users.append(User(**cleanedUser)) # Filter by mandate if specified (via UserMandate table) if mandateId: diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 071f8673..3a3e3ab4 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -15,7 +15,7 @@ import json # Import interfaces and models from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelChatbot import AutomationDefinition, ChatWorkflow +from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.features.workflow import executeAutomation diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index bf3fbc73..817ab762 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -321,11 +321,11 @@ async def delete_mandate( # User Management within Mandates (Mandate-Admin) # ============================================================================= -@router.get("/{mandateId}/users", response_model=List[MandateUserInfo]) +@router.get("/{targetMandateId}/users", response_model=List[MandateUserInfo]) @limiter.limit("60/minute") async def listMandateUsers( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), context: RequestContext = Depends(getRequestContext) ) -> List[MandateUserInfo]: """ @@ -334,7 +334,7 @@ async def listMandateUsers( Requires Mandate-Admin role or SysAdmin. """ # Check permission - if not _hasMandateAdminRole(context, mandateId) and not context.isSysAdmin: + if not _hasMandateAdminRole(context, targetMandateId) and not context.isSysAdmin: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -344,17 +344,17 @@ async def listMandateUsers( rootInterface = interfaceDbAppObjects.getRootInterface() # Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # Get all UserMandate entries for this mandate userMandates = rootInterface.db.getRecordset( UserMandate, - recordFilter={"mandateId": mandateId} + recordFilter={"mandateId": targetMandateId} ) result = [] @@ -383,18 +383,18 @@ async def listMandateUsers( except HTTPException: raise except Exception as e: - logger.error(f"Error listing users for mandate {mandateId}: {e}") + logger.error(f"Error listing users for mandate {targetMandateId}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to list users: {str(e)}" ) -@router.post("/{mandateId}/users", response_model=UserMandateResponse) +@router.post("/{targetMandateId}/users", response_model=UserMandateResponse) @limiter.limit("30/minute") async def addUserToMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), data: UserMandateCreate = Body(...), context: RequestContext = Depends(getRequestContext) ) -> UserMandateResponse: @@ -405,7 +405,7 @@ async def addUserToMandate( SysAdmin cannot add themselves (Self-Eskalation Prevention). Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID data: User ID and role IDs to assign """ # 1. SysAdmin Self-Eskalation Prevention @@ -416,7 +416,7 @@ async def addUserToMandate( ) # 2. Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required to add users" @@ -426,11 +426,11 @@ async def addUserToMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # 3. Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # 4. Verify target user exists @@ -442,7 +442,7 @@ async def addUserToMandate( ) # 5. Check if user is already a member - existingMembership = rootInterface.getUserMandate(data.targetUserId, mandateId) + existingMembership = rootInterface.getUserMandate(data.targetUserId, targetMandateId) if existingMembership: raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -459,7 +459,7 @@ async def addUserToMandate( ) role = roleRecords[0] roleMandateId = role.get("mandateId") - if roleMandateId and str(roleMandateId) != str(mandateId): + if roleMandateId and str(roleMandateId) != str(targetMandateId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Role {roleId} belongs to a different mandate" @@ -468,14 +468,14 @@ async def addUserToMandate( # 7. Create UserMandate userMandate = rootInterface.createUserMandate( userId=data.targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=data.roleIds ) # 8. Audit - Log permission change with IP address audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="user_added_to_mandate", targetUserId=data.targetUserId, details=f"Roles assigned: {data.roleIds}", @@ -484,14 +484,14 @@ async def addUserToMandate( ) logger.info( - f"User {context.user.id} added user {data.targetUserId} to mandate {mandateId} " + f"User {context.user.id} added user {data.targetUserId} to mandate {targetMandateId} " f"with roles {data.roleIds}" ) return UserMandateResponse( userMandateId=str(userMandate.id), userId=data.targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=data.roleIds, enabled=True ) @@ -506,11 +506,11 @@ async def addUserToMandate( ) -@router.delete("/{mandateId}/users/{targetUserId}", response_model=Dict[str, str]) +@router.delete("/{targetMandateId}/users/{targetUserId}", response_model=Dict[str, str]) @limiter.limit("30/minute") async def removeUserFromMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user to remove"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, str]: @@ -521,11 +521,11 @@ async def removeUserFromMandate( Cannot remove the last admin from a mandate (orphan prevention). Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID targetUserId: User ID to remove """ # Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -535,15 +535,15 @@ async def removeUserFromMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # Get user's membership - membership = rootInterface.getUserMandate(targetUserId, mandateId) + membership = rootInterface.getUserMandate(targetUserId, targetMandateId) if not membership: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -551,26 +551,26 @@ async def removeUserFromMandate( ) # Check if this is the last admin (orphan prevention) - if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + if _isLastMandateAdmin(rootInterface, targetMandateId, targetUserId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot remove the last admin from a mandate. Assign another admin first." ) # Delete UserMandate (CASCADE will delete UserMandateRole entries) - rootInterface.deleteUserMandate(targetUserId, mandateId) + rootInterface.deleteUserMandate(targetUserId, targetMandateId) # Audit - Log permission change audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="user_removed_from_mandate", targetUserId=targetUserId, details="User removed from mandate", resourceType="UserMandate" ) - logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {mandateId}") + logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {targetMandateId}") return {"message": "User removed from mandate", "userId": targetUserId} @@ -584,11 +584,11 @@ async def removeUserFromMandate( ) -@router.put("/{mandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) +@router.put("/{targetMandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) @limiter.limit("30/minute") async def updateUserRolesInMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user"), roleIds: List[str] = Body(..., description="New role IDs to assign"), context: RequestContext = Depends(getRequestContext) @@ -600,12 +600,12 @@ async def updateUserRolesInMandate( Requires Mandate-Admin role. Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID targetUserId: User ID to update roleIds: New set of role IDs """ # Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -615,7 +615,7 @@ async def updateUserRolesInMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # Get user's membership - membership = rootInterface.getUserMandate(targetUserId, mandateId) + membership = rootInterface.getUserMandate(targetUserId, targetMandateId) if not membership: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -632,7 +632,7 @@ async def updateUserRolesInMandate( ) role = roleRecords[0] roleMandateId = role.get("mandateId") - if roleMandateId and str(roleMandateId) != str(mandateId): + if roleMandateId and str(roleMandateId) != str(targetMandateId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Role {roleId} belongs to a different mandate" @@ -640,11 +640,11 @@ async def updateUserRolesInMandate( # Check if removing admin role would leave mandate without admins currentRoleIds = rootInterface.getRoleIdsForUserMandate(str(membership.id)) - isCurrentlyAdmin = _hasAdminRoleInList(rootInterface, currentRoleIds, mandateId) - willBeAdmin = _hasAdminRoleInList(rootInterface, roleIds, mandateId) + isCurrentlyAdmin = _hasAdminRoleInList(rootInterface, currentRoleIds, targetMandateId) + willBeAdmin = _hasAdminRoleInList(rootInterface, roleIds, targetMandateId) if isCurrentlyAdmin and not willBeAdmin: - if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + if _isLastMandateAdmin(rootInterface, targetMandateId, targetUserId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot remove admin role from the last admin. Assign another admin first." @@ -665,7 +665,7 @@ async def updateUserRolesInMandate( # Audit - Log role assignment change audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="role_assigned", targetUserId=targetUserId, details=f"New roles: {roleIds}", @@ -675,13 +675,13 @@ async def updateUserRolesInMandate( logger.info( f"User {context.user.id} updated roles for user {targetUserId} " - f"in mandate {mandateId} to {roleIds}" + f"in mandate {targetMandateId} to {roleIds}" ) return UserMandateResponse( userMandateId=str(membership.id), userId=targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=roleIds, enabled=membership.enabled ) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 226f3606..a9ebbab5 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -71,16 +71,43 @@ async def get_users( # MULTI-TENANT: Use mandateId from context (header) # SysAdmin without mandateId can see all users if context.mandateId: - # Get users for specific mandate via UserMandate table - from modules.datamodels.datamodelMembership import UserMandate - userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": str(context.mandateId)}) - userIds = [str(um["userId"]) for um in userMandates] + # Get users for specific mandate using getUsersByMandate + result = appInterface.getUsersByMandate(str(context.mandateId), paginationParams) - # Get all users and filter by mandate membership - allUsers = appInterface.getUsers() - users = [u for u in allUsers if str(u.id) in userIds] + # getUsersByMandate returns PaginatedResult if pagination was provided + if paginationParams and hasattr(result, 'items'): + return PaginatedResponse( + items=result.items, + pagination=PaginationMetadata( + currentPage=result.currentPage, + pageSize=result.pageSize, + totalItems=result.totalItems, + totalPages=result.totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + else: + # No pagination - result is a list + users = result if isinstance(result, list) else result.items if hasattr(result, 'items') else [] + return PaginatedResponse( + items=users, + pagination=None + ) + elif context.isSysAdmin: + # SysAdmin without mandateId sees all users + # Get all users directly from database using UserInDB (the actual database model) + from modules.datamodels.datamodelUam import UserInDB + allUsers = appInterface.db.getRecordset(UserInDB) + # Convert to User objects, filtering out password hash and database-specific fields + users = [] + for u in allUsers: + cleanedUser = {k: v for k, v in u.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} + # Ensure roleLabels is always a list + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + users.append(User(**cleanedUser)) - # Apply pagination manually if needed if paginationParams: totalItems = len(users) import math @@ -105,33 +132,6 @@ async def get_users( items=users, pagination=None ) - elif context.isSysAdmin: - # SysAdmin without mandateId sees all users - result = appInterface.getUsers() - if paginationParams: - totalItems = len(result) - import math - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - paginatedUsers = result[startIdx:endIdx] - - return PaginatedResponse( - items=paginatedUsers, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters - ) - ) - else: - return PaginatedResponse( - items=result, - pagination=None - ) else: # Non-SysAdmin without mandateId - should not happen (getRequestContext enforces) raise HTTPException( diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index d6f53d84..ed1fd9f3 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -16,7 +16,7 @@ from modules.auth import limiter, getRequestContext, RequestContext import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot # Import models -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions from modules.features.workflow import chatStart, chatStop diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index 20b90876..b5b80e2e 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -22,7 +22,7 @@ import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 4d724218..01e33eac 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -89,6 +89,212 @@ async def listFeatures( ) +# ============================================================================= +# My Feature Instances (No mandate context needed) +# IMPORTANT: Must be before /{featureCode} to avoid route matching conflict +# ============================================================================= + +class FeaturesMyResponse(BaseModel): + """Hierarchical response for GET /features/my""" + mandates: List[Dict[str, Any]] + + +@router.get("/my", response_model=FeaturesMyResponse) +@limiter.limit("60/minute") +async def getMyFeatureInstances( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> FeaturesMyResponse: + """ + Get all feature instances the current user has access to. + + Returns hierarchical structure: mandates -> features -> instances -> permissions + This endpoint does not require X-Mandate-Id header. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Get all feature accesses for this user + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + + if not featureAccesses: + return FeaturesMyResponse(mandates=[]) + + # Build hierarchical structure: mandate -> feature -> instances + mandatesMap: Dict[str, Dict[str, Any]] = {} + featuresMap: Dict[str, Dict[str, Any]] = {} # key: mandateId_featureCode + + for access in featureAccesses: + if not access.enabled: + continue + + instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) + if not instance or not instance.enabled: + continue + + # Get mandate info + mandateId = str(instance.mandateId) + if mandateId not in mandatesMap: + mandate = rootInterface.getMandate(mandateId) + if mandate: + mandatesMap[mandateId] = { + "id": mandateId, + "name": mandate.name if hasattr(mandate, 'name') else mandateId, + "code": mandate.code if hasattr(mandate, 'code') else None, + "features": [] + } + else: + mandatesMap[mandateId] = { + "id": mandateId, + "name": mandateId, + "code": None, + "features": [] + } + + # Get feature info + featureKey = f"{mandateId}_{instance.featureCode}" + if featureKey not in featuresMap: + feature = featureInterface.getFeature(instance.featureCode) + featuresMap[featureKey] = { + "code": instance.featureCode, + "label": feature.label if feature and hasattr(feature, 'label') else {"de": instance.featureCode, "en": instance.featureCode}, + "icon": feature.icon if feature and hasattr(feature, 'icon') else "folder", + "instances": [], + "_mandateId": mandateId # Temporary for grouping + } + + # Get user's role in this instance + userRole = _getUserRoleInInstance(rootInterface, str(context.user.id), str(instance.id)) + + # Get permissions for this instance + permissions = _getInstancePermissions(rootInterface, str(context.user.id), str(instance.id)) + + # Add instance to feature + featuresMap[featureKey]["instances"].append({ + "id": str(instance.id), + "featureCode": instance.featureCode, + "mandateId": mandateId, + "mandateName": mandatesMap[mandateId]["name"], + "instanceLabel": instance.label, + "userRole": userRole, + "permissions": permissions + }) + + # Build final structure + for featureKey, featureData in featuresMap.items(): + mandateId = featureData.pop("_mandateId") + mandatesMap[mandateId]["features"].append(featureData) + + return FeaturesMyResponse(mandates=list(mandatesMap.values())) + + except Exception as e: + logger.error(f"Error getting user's feature instances: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature instances: {str(e)}" + ) + + +def _getUserRoleInInstance(rootInterface, userId: str, instanceId: str) -> str: + """Get the user's primary role label in a feature instance.""" + try: + from modules.datamodels.datamodelRbac import UserRole, Role + + # Get user-role assignments for this instance + userRoles = rootInterface.db.getRecordset( + UserRole, + recordFilter={"userId": userId} + ) + + for ur in userRoles: + roleId = ur.get("roleId") + if roleId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles and str(roles[0].get("featureInstanceId")) == instanceId: + return roles[0].get("roleLabel", "user") + + return "user" # Default + except Exception as e: + logger.debug(f"Error getting user role: {e}") + return "user" + + +def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict[str, Any]: + """Get summarized permissions for a user in an instance.""" + # Default permissions structure + permissions = { + "tables": {}, + "views": {}, + "fields": {} + } + + try: + from modules.datamodels.datamodelRbac import UserRole, Role, RolePermission + + # Get user's roles for this instance + userRoles = rootInterface.db.getRecordset(UserRole, recordFilter={"userId": userId}) + roleIds = [] + + for ur in userRoles: + roleId = ur.get("roleId") + if roleId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles and str(roles[0].get("featureInstanceId")) == instanceId: + roleIds.append(roleId) + + if not roleIds: + return permissions + + # Get permissions for all roles + for roleId in roleIds: + rolePerms = rootInterface.db.getRecordset( + RolePermission, + recordFilter={"roleId": roleId} + ) + + for perm in rolePerms: + tableName = perm.get("tableName", "") + if tableName: + if tableName not in permissions["tables"]: + permissions["tables"][tableName] = { + "view": False, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + } + + # Merge permissions (highest wins) + current = permissions["tables"][tableName] + current["view"] = current["view"] or perm.get("canView", False) + current["read"] = _mergeAccessLevel(current["read"], perm.get("readLevel", "n")) + current["create"] = _mergeAccessLevel(current["create"], perm.get("createLevel", "n")) + current["update"] = _mergeAccessLevel(current["update"], perm.get("updateLevel", "n")) + current["delete"] = _mergeAccessLevel(current["delete"], perm.get("deleteLevel", "n")) + + viewName = perm.get("viewName", "") + if viewName: + permissions["views"][viewName] = permissions["views"].get(viewName, False) or perm.get("canAccess", False) + + return permissions + + except Exception as e: + logger.debug(f"Error getting instance permissions: {e}") + return permissions + + +def _mergeAccessLevel(current: str, new: str) -> str: + """Merge two access levels, returning the highest.""" + levels = {"n": 0, "m": 1, "g": 2, "a": 3} + currentLevel = levels.get(current, 0) + newLevel = levels.get(new, 0) + + if newLevel > currentLevel: + return new + return current + + @router.get("/{featureCode}", response_model=Dict[str, Any]) @limiter.limit("60/minute") async def getFeature( @@ -539,55 +745,6 @@ async def createTemplateRole( ) -# ============================================================================= -# My Feature Instances (No mandate context needed) -# ============================================================================= - -@router.get("/my", response_model=List[Dict[str, Any]]) -@limiter.limit("60/minute") -async def getMyFeatureInstances( - request: Request, - context: RequestContext = Depends(getRequestContext) -) -> List[Dict[str, Any]]: - """ - Get all feature instances the current user has access to. - - Returns instances across all mandates the user is member of. - This endpoint does not require X-Mandate-Id header. - """ - try: - rootInterface = getRootInterface() - - # Get all feature accesses for this user - featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) - - if not featureAccesses: - return [] - - featureInterface = getFeatureInterface(rootInterface.db) - result = [] - - for access in featureAccesses: - if not access.enabled: - continue - - instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) - if instance and instance.enabled: - result.append({ - **instance.model_dump(), - "accessId": str(access.id) - }) - - return result - - except Exception as e: - logger.error(f"Error getting user's feature instances: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get feature instances: {str(e)}" - ) - - # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 1c7d9e80..ab9e1ff6 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -19,7 +19,7 @@ from modules.interfaces.interfaceDbChatbot import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( ChatWorkflow, ChatMessage, ChatLog, diff --git a/modules/services/__init__.py b/modules/services/__init__.py index b75b2454..fb4e6512 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -3,7 +3,7 @@ from typing import Any, Optional from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow class PublicService: """Lightweight proxy exposing only public callable attributes of a target. diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 55bd2544..cd86c6a8 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -6,7 +6,7 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChatbot import PromptPlaceholder, ChatDocument +from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py index ec6a26d2..a866f68f 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py index e90ecfeb..821851a4 100644 --- a/modules/services/serviceAi/subDocumentIntents.py +++ b/modules/services/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 0c82929d..137dcd05 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChatbot import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 64678f54..13739dea 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -11,7 +11,7 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index adc0ea78..a49b78c7 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -6,7 +6,7 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.services.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, diff --git a/modules/shared/dbMultiTenantOptimizations.py b/modules/shared/dbMultiTenantOptimizations.py index 38f154a5..3e056179 100644 --- a/modules/shared/dbMultiTenantOptimizations.py +++ b/modules/shared/dbMultiTenantOptimizations.py @@ -21,6 +21,22 @@ from typing import Optional, List logger = logging.getLogger(__name__) +def _getConnection(dbConnector): + """Get a connection from the DatabaseConnector. + + Ensures the connection is alive and returns it. + Commits any pending transaction first to avoid blocking. + """ + dbConnector._ensure_connection() + conn = dbConnector.connection + # Commit any pending transaction to avoid blocking + try: + conn.commit() + except Exception: + pass # Ignore if nothing to commit + return conn + + # ============================================================================= # Index Definitions # ============================================================================= @@ -144,28 +160,45 @@ def applyMultiTenantOptimizations(dbConnector, tables: Optional[List[str]] = Non try: # Get a connection from the connector - conn = dbConnector._get_connection() - conn.autocommit = True + conn = _getConnection(dbConnector) - with conn.cursor() as cursor: - # Apply indexes - results["indexesCreated"] = _applyIndexes(cursor, tables) - - # Apply foreign keys - results["foreignKeysCreated"] = _applyForeignKeys(cursor, tables) - - # Apply immutable triggers - results["triggersCreated"] = _applyImmutableTriggers(cursor, tables) + # Save and set autocommit state + try: + originalAutocommit = conn.autocommit + except Exception: + originalAutocommit = False - logger.info( - f"Multi-tenant optimizations applied: " - f"{results['indexesCreated']} indexes, " - f"{results['triggersCreated']} triggers, " - f"{results['foreignKeysCreated']} foreign keys" - ) + try: + conn.autocommit = True + except Exception as autoErr: + logger.debug(f"Could not set autocommit: {autoErr}") + + try: + with conn.cursor() as cursor: + # Apply indexes + results["indexesCreated"] = _applyIndexes(cursor, tables) + + # Apply foreign keys + results["foreignKeysCreated"] = _applyForeignKeys(cursor, tables) + + # Apply immutable triggers + results["triggersCreated"] = _applyImmutableTriggers(cursor, tables) + + logger.info( + f"Multi-tenant optimizations applied: " + f"{results['indexesCreated']} indexes, " + f"{results['triggersCreated']} triggers, " + f"{results['foreignKeysCreated']} foreign keys" + ) + finally: + # Restore original autocommit state + try: + conn.autocommit = originalAutocommit + except Exception: + pass except Exception as e: - logger.error(f"Error applying multi-tenant optimizations: {e}") + logger.error(f"Error applying multi-tenant optimizations: {type(e).__name__}: {e}") results["errors"].append(str(e)) return results @@ -174,11 +207,15 @@ def applyMultiTenantOptimizations(dbConnector, tables: Optional[List[str]] = Non def applyIndexesOnly(dbConnector, tables: Optional[List[str]] = None) -> int: """Apply only indexes (lighter operation, safe for frequent calls).""" try: - conn = dbConnector._get_connection() + conn = _getConnection(dbConnector) + originalAutocommit = conn.autocommit conn.autocommit = True - with conn.cursor() as cursor: - return _applyIndexes(cursor, tables) + try: + with conn.cursor() as cursor: + return _applyIndexes(cursor, tables) + finally: + conn.autocommit = originalAutocommit except Exception as e: logger.error(f"Error applying indexes: {e}") return 0 @@ -194,9 +231,13 @@ def _tableExists(cursor, tableName: str) -> bool: SELECT EXISTS ( SELECT FROM information_schema.tables WHERE table_name = %s - ) + ) AS exists """, (tableName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + # Handle both dict (RealDictCursor) and tuple results + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _indexExists(cursor, indexName: str) -> bool: @@ -205,9 +246,12 @@ def _indexExists(cursor, indexName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_indexes WHERE indexname = %s - ) + ) AS exists """, (indexName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _constraintExists(cursor, constraintName: str) -> bool: @@ -216,9 +260,12 @@ def _constraintExists(cursor, constraintName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_constraint WHERE conname = %s - ) + ) AS exists """, (constraintName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _triggerExists(cursor, triggerName: str) -> bool: @@ -227,9 +274,12 @@ def _triggerExists(cursor, triggerName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_trigger WHERE tgname = %s - ) + ) AS exists """, (triggerName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _applyIndexes(cursor, tables: Optional[List[str]]) -> int: @@ -390,7 +440,7 @@ def getOptimizationStatus(dbConnector) -> dict: } try: - conn = dbConnector._get_connection() + conn = _getConnection(dbConnector) with conn.cursor() as cursor: # Check regular indexes for tableName, indexName, _ in _INDEXES: diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index a3f45261..39d6e16f 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 77cb361f..4f9bbd21 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 6c509a9e..65e95a32 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index bddeb252..f804c0b9 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index 806679df..e32c1965 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index 96f2609c..bb6f8437 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 4c5d8314..62b43bce 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index c6e6d560..ff7e896f 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index cccad45d..5b90ce13 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index fedbc46b..9991285b 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 0b646251..8e3b7185 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index 015eb1e3..2f011a25 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index f00192f6..45b60cad 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index 1e44b1cb..cbec7960 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index afa0c5fc..631795b3 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 6e6106b0..55d99654 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index d72e3d55..b997889e 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 49ddece2..2bd7ab74 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index 24c097e8..bbdc2cc7 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index adfe13ea..5ac4e548 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 06f26e89..59604896 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 4ff700ca..2d325d9f 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index 4531859f..f8831d59 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index da9f8cd4..9b7fb011 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index 05997512..a4bf18b6 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index 287612ff..f149e482 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index e6c2a276..c64a6637 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 4eac8544..722dbc99 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index a9b837aa..62b6dd94 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index d0838633..318271c3 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index eaf3254f..73cdb730 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index ddce6206..e9361853 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 85d7b123..1f469b80 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 7bde4da7..0e4d6ee4 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionItem, TaskStep -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index 0daac228..a4ae05e9 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 4b1fcad5..0fac427c 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.datamodels.datamodelChatbot import WorkflowModeEnum + from modules.datamodels.datamodelChat import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index 85f1f824..e3131939 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index e8837d65..770c868a 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskResult, ActionItem -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index b821511b..f7754eab 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.datamodels.datamodelChatbot import ReviewContext + from modules.datamodels.datamodelChat import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.datamodels.datamodelChatbot import ReviewResult + from modules.datamodels.datamodelChat import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index b0186be9..1cdf0d53 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.datamodels.datamodelChatbot import TaskStep, ActionResult, Observation +from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 30e9af4d..a6e3a78a 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChatbot import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChatbot import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 10932529..31878033 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index e1d767c4..11a54ca1 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 317a6cb7..9c9d6c84 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -6,9 +6,9 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING -from modules.datamodels import datamodelChatbot -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum +from modules.datamodels import datamodelChat +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -102,7 +102,7 @@ class WorkflowProcessor: self.services.chat.progressLogFinish(operationId, False) raise - async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChatbot.TaskResult: + async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChat.TaskResult: """Execute a task step using the appropriate mode""" import time @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.datamodels.datamodelChatbot import ActionDocument + from modules.datamodels.datamodelChat import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.datamodels.datamodelChatbot import ChatMessage, ChatDocument, ActionDocument + from modules.datamodels.datamodelChat import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index e6ecdbbd..a9b656eb 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.datamodels.datamodelChatbot import TaskContext +from modules.datamodels.datamodelChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.datamodels.datamodelChatbot import ActionResult + from modules.datamodels.datamodelChat import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 94eb6158..12a374f8 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -85,7 +85,7 @@ class AIModelsTester: self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index dd5d68e3..05a3f34b 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.datamodels.datamodelChatbot import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.datamodels.datamodelChatbot import ChatMessage + from modules.datamodels.datamodelChat import ChatMessage import time testMessage = ChatMessage( diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 478e9baf..9da22d66 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,7 +42,7 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid import time import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 3a0cf2a3..7beaeec4 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,7 +20,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 784b1756..698c9698 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,7 +22,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 9c96d95f..d9c4d9b8 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,7 +21,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index a0b744f4..45a364ce 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,7 +21,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index e1331cd1..6d1735ad 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,7 +23,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index 9409a9e6..a2b69576 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index b91aa1e7..ae502397 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 25a5af8e..09f6e92c 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel diff --git a/tool_db_adapt_to_models.py b/tool_db_adapt_to_models.py new file mode 100644 index 00000000..85c6e8fc --- /dev/null +++ b/tool_db_adapt_to_models.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +""" +Datenbank-Anpassung an Pydantic-Modelle. + +Einfaches Script das: +1. Fehlende Felder in DB ergänzt (gemäss Pydantic-Modellen) +2. Falsche Datentypen korrigiert (Daten werden ggf. gelöscht) +3. Spezialfall: UserInDB.privilege → roleLabels migriert + +Verwendung: + python tool_db_adapt_to_models.py [--dry-run] [--db ] +""" + +import os +import sys +import json +import argparse +import logging +from pathlib import Path +from typing import Dict, List, Any, Optional + +# Gateway-Pfad setzen +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent +sys.path.insert(0, str(gatewayPath)) +os.chdir(str(gatewayPath)) + +# Logging ZUERST konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + force=True # Überschreibt bestehende Config +) +logger = logging.getLogger(__name__) + +import psycopg2 +import psycopg2.extras +from modules.shared.configuration import APP_CONFIG + +# Datenbank-Konfiguration: DB-Name → (Config-Prefix, Pydantic-Modelle) +DATABASE_CONFIG = { + "poweron_app": ("DB_APP", ["datamodelUam", "datamodelRbac", "datamodelSecurity"]), + "poweron_chat": ("DB_CHAT", ["datamodelChat"]), + "poweron_management": ("DB_MANAGEMENT", ["datamodelWorkflow", "datamodelFiles"]), +} + +# Python-Typ → PostgreSQL-Typ Mapping +TYPE_MAPPING = { + "str": "text", + "int": "integer", + "float": "double precision", + "bool": "boolean", + "list": "jsonb", + "dict": "jsonb", + "List": "jsonb", + "Dict": "jsonb", + "Optional": None, # Wird separat behandelt + "datetime": "timestamp", + "EmailStr": "text", + "UUID": "uuid", +} + + +def _getDbConnection(dbName: str): + """Verbindet mit einer Datenbank über APP_CONFIG.""" + prefix = DATABASE_CONFIG.get(dbName, ("DB", []))[0] + + host = APP_CONFIG.get(f"{prefix}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") + port = APP_CONFIG.get(f"{prefix}_PORT") or APP_CONFIG.get("DB_PORT", "5432") + user = APP_CONFIG.get(f"{prefix}_USER") or APP_CONFIG.get("DB_USER") + password = APP_CONFIG.get(f"{prefix}_PASSWORD_SECRET") or APP_CONFIG.get(f"{prefix}_PASSWORD") or APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD") + + if not user or not password: + logger.error(f"Keine Credentials für {dbName} ({prefix}_*)") + return None + + try: + conn = psycopg2.connect( + host=host, port=int(port), database=dbName, + user=user, password=password, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.autocommit = True + return conn + except Exception as e: + logger.error(f"Verbindungsfehler {dbName}: {e}") + return None + + +def _getDbTables(conn) -> Dict[str, Dict[str, str]]: + """Holt alle Tabellen und deren Spalten aus der DB.""" + cur = conn.cursor() + cur.execute(""" + SELECT table_name, column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = 'public' + ORDER BY table_name, ordinal_position + """) + + tables = {} + for row in cur.fetchall(): + tableName = row['table_name'] + if tableName not in tables: + tables[tableName] = {} + tables[tableName][row['column_name']] = { + 'type': row['data_type'], + 'nullable': row['is_nullable'] == 'YES' + } + cur.close() + return tables + + +def _parsePydanticModels(moduleNames: List[str]) -> Dict[str, Dict[str, str]]: + """Parst Pydantic-Modelle aus den angegebenen Modulen.""" + import ast + + models = {} + datamodelsPath = gatewayPath / "modules" / "datamodels" + + for moduleName in moduleNames: + filePath = datamodelsPath / f"{moduleName}.py" + if not filePath.exists(): + logger.warning(f"Modul nicht gefunden: {filePath}") + continue + + with open(filePath, 'r', encoding='utf-8') as f: + tree = ast.parse(f.read()) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + className = node.name + + # Prüfe ob Klasse von BaseModel erbt + isBaseModel = False + for base in node.bases: + if isinstance(base, ast.Name) and base.id == "BaseModel": + isBaseModel = True + break + + if not isBaseModel: + continue + + fields = {} + for item in node.body: + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name): + fieldName = item.target.id + if fieldName.startswith('_'): + continue + + # Typ extrahieren + fieldType = _extractType(item.annotation) + if fieldType: + fields[fieldName] = fieldType + + if fields: + models[className] = fields + + return models + + +def _extractType(annotation) -> Optional[str]: + """Extrahiert den PostgreSQL-Typ aus einer AST-Annotation.""" + import ast + + if isinstance(annotation, ast.Name): + return TYPE_MAPPING.get(annotation.id, "text") + + elif isinstance(annotation, ast.Subscript): + # Optional[X], List[X], Dict[X, Y] + if isinstance(annotation.value, ast.Name): + outerType = annotation.value.id + if outerType == "Optional": + # Rekursiv den inneren Typ holen + if isinstance(annotation.slice, ast.Name): + return TYPE_MAPPING.get(annotation.slice.id, "text") + elif isinstance(annotation.slice, ast.Subscript): + return _extractType(annotation.slice) + return "text" + elif outerType in ("List", "list", "Dict", "dict"): + return "jsonb" + + elif isinstance(annotation, ast.Constant): + return "text" + + return "text" + + +def _adaptTable(conn, tableName: str, modelFields: Dict[str, str], dbColumns: Dict[str, Any], dryRun: bool) -> bool: + """Passt eine Tabelle an das Pydantic-Modell an.""" + cur = conn.cursor() + success = True + + for fieldName, pgType in modelFields.items(): + # pgType kommt direkt aus _extractType (ist bereits PostgreSQL-Typ) + + # Suche Spalte case-insensitive + dbCol = None + actualColName = None + for colName, colInfo in dbColumns.items(): + if colName.lower() == fieldName.lower(): + dbCol = colInfo + actualColName = colName + break + + if dbCol is None: + # Spalte fehlt → hinzufügen + sql = f'ALTER TABLE "{tableName}" ADD COLUMN "{fieldName}" {pgType}' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + else: + try: + cur.execute(sql) + logger.info(f"Spalte hinzugefügt: {tableName}.{fieldName} ({pgType})") + except Exception as e: + logger.error(f"Fehler beim Hinzufügen von {tableName}.{fieldName}: {e}") + success = False + else: + # Spalte existiert → Typ prüfen + currentType = dbCol['type'] + if not _typesCompatible(currentType, pgType): + sql = f'ALTER TABLE "{tableName}" ALTER COLUMN "{actualColName}" TYPE {pgType} USING NULL' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + else: + try: + cur.execute(sql) + logger.info(f"Typ geändert: {tableName}.{actualColName} ({currentType} → {pgType})") + except Exception as e: + logger.error(f"Fehler beim Ändern von {tableName}.{actualColName}: {e}") + success = False + + cur.close() + return success + + +def _typesCompatible(dbType: str, targetType: str) -> bool: + """Prüft ob DB-Typ mit Ziel-Typ kompatibel ist.""" + dbType = dbType.lower() + targetType = targetType.lower() + + # Gleiche Typen + if dbType == targetType: + return True + + # Kompatible Typen + compatiblePairs = [ + ("character varying", "text"), + ("varchar", "text"), + ("integer", "bigint"), + ("real", "double precision"), + ("timestamp without time zone", "timestamp"), + ("timestamp with time zone", "timestamp"), + ] + + for a, b in compatiblePairs: + if (dbType == a and targetType == b) or (dbType == b and targetType == a): + return True + + return False + + +def _migratePrivilegeToRoleLabels(conn, tableName: str, dryRun: bool) -> bool: + """Migriert privilege-Wert nach roleLabels (Spezialfall UserInDB).""" + cur = conn.cursor() + + # Prüfe ob beide Spalten existieren + cur.execute(""" + SELECT column_name FROM information_schema.columns + WHERE table_name = %s AND column_name IN ('privilege', 'roleLabels') + """, (tableName,)) + columns = [row['column_name'] for row in cur.fetchall()] + + if 'privilege' not in columns: + logger.info(f"Spalte 'privilege' existiert nicht in {tableName} - keine Migration nötig") + cur.close() + return True + + if 'roleLabels' not in columns: + # roleLabels erstellen + sql = f'ALTER TABLE "{tableName}" ADD COLUMN "roleLabels" jsonb' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + cur.close() + return True + else: + cur.execute(sql) + logger.info(f"Spalte roleLabels in {tableName} erstellt") + + # Migriere privilege → roleLabels + cur.execute(f""" + SELECT id, privilege, "roleLabels" FROM "{tableName}" + WHERE privilege IS NOT NULL AND privilege != '' + """) + users = cur.fetchall() + + if not users: + logger.info(f"Keine Einträge mit privilege-Wert in {tableName}") + cur.close() + return True + + logger.info(f"Migriere {len(users)} Einträge: privilege → roleLabels") + + for user in users: + userId = user['id'] + privilege = user['privilege'] + roleLabels = user['roleLabels'] or [] + + if isinstance(roleLabels, str): + try: + roleLabels = json.loads(roleLabels) + except: + roleLabels = [] + + if privilege not in roleLabels: + roleLabels.append(privilege) + + if dryRun: + logger.info(f"[DRY-RUN] UPDATE {tableName} SET roleLabels = {roleLabels} WHERE id = {userId}") + else: + cur.execute( + f'UPDATE "{tableName}" SET "roleLabels" = %s WHERE id = %s', + (json.dumps(roleLabels), userId) + ) + + if not dryRun: + logger.info(f"Migration privilege → roleLabels abgeschlossen") + + cur.close() + return True + + +def runMigration(dbName: str, dryRun: bool) -> bool: + """Führt Migration für eine Datenbank durch.""" + logger.info(f"\n{'='*60}") + logger.info(f"DATENBANK: {dbName}") + logger.info(f"{'='*60}") + + if dbName not in DATABASE_CONFIG: + logger.error(f"Unbekannte Datenbank: {dbName}") + return False + + conn = _getDbConnection(dbName) + if not conn: + return False + + prefix, moduleNames = DATABASE_CONFIG[dbName] + + # DB-Schema laden + dbTables = _getDbTables(conn) + logger.info(f"Tabellen in DB: {', '.join(dbTables.keys()) if dbTables else 'keine'}") + + # Pydantic-Modelle laden + models = _parsePydanticModels(moduleNames) + logger.info(f"Pydantic-Modelle: {', '.join(models.keys()) if models else 'keine'}") + + success = True + + # Jedes Modell mit passender DB-Tabelle abgleichen + for modelName, modelFields in models.items(): + # Finde passende Tabelle (case-insensitive) + tableName = None + tableColumns = None + for dbTable, dbCols in dbTables.items(): + if dbTable.lower() == modelName.lower(): + tableName = dbTable + tableColumns = dbCols + break + + if tableName is None: + logger.warning(f"Keine Tabelle für Modell {modelName} gefunden - übersprungen") + continue + + logger.info(f"\nPrüfe {modelName} ↔ {tableName}...") + + # Tabelle anpassen + if not _adaptTable(conn, tableName, modelFields, tableColumns, dryRun): + success = False + + # Spezialfall: UserInDB privilege → roleLabels + if modelName == "UserInDB": + if not _migratePrivilegeToRoleLabels(conn, tableName, dryRun): + success = False + + conn.close() + return success + + +def main(): + parser = argparse.ArgumentParser(description="Passt DB-Struktur an Pydantic-Modelle an") + parser.add_argument("--dry-run", action="store_true", help="Zeigt nur geplante Änderungen") + parser.add_argument("--db", help="Nur bestimmte DB(s) migrieren (komma-getrennt)") + args = parser.parse_args() + + databases = list(DATABASE_CONFIG.keys()) + if args.db: + databases = [db.strip() for db in args.db.split(",")] + + logger.info("="*60) + logger.info("DATENBANK-ANPASSUNG AN PYDANTIC-MODELLE") + logger.info("="*60) + logger.info(f"Modus: {'DRY-RUN' if args.dry_run else 'LIVE'}") + logger.info(f"Datenbanken: {', '.join(databases)}") + + if not args.dry_run: + response = input("\nÄnderungen durchführen? (yes/no): ") + if response.lower() != "yes": + logger.info("Abgebrochen") + return 0 + + allSuccess = True + for dbName in databases: + if not runMigration(dbName, args.dry_run): + allSuccess = False + + if allSuccess: + logger.info("\n" + "="*60) + logger.info("ALLE MIGRATIONEN ERFOLGREICH") + logger.info("="*60) + else: + logger.error("\n" + "="*60) + logger.error("EINIGE MIGRATIONEN HATTEN FEHLER") + logger.error("="*60) + + return 0 if allSuccess else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tool_db_export_migration.py b/tool_db_export_migration.py index 6c93370a..aea697c1 100644 --- a/tool_db_export_migration.py +++ b/tool_db_export_migration.py @@ -36,9 +36,36 @@ from datetime import datetime from typing import Dict, List, Any, Optional from pathlib import Path +# Add gateway to path for imports and set working directory +# Find gateway directory (could be in local/pending/ or gateway/) +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent +# If we're in local/pending/, go up to find gateway/ +if gatewayPath.name == "pending": + gatewayPath = gatewayPath.parent.parent / "gateway" +elif gatewayPath.name == "local": + gatewayPath = gatewayPath.parent / "gateway" +# If gateway doesn't exist, try current directory +if not gatewayPath.exists(): + gatewayPath = Path(__file__).parent.parent.parent / "gateway" +if gatewayPath.exists(): + sys.path.insert(0, str(gatewayPath)) + # Change working directory to gateway so APP_CONFIG can find .env file + os.chdir(str(gatewayPath)) +else: + # Fallback: assume we're already in gateway/ or add parent + sys.path.insert(0, str(Path(__file__).parent)) + # Try to change to gateway directory if it exists + potentialGateway = Path(__file__).parent + if potentialGateway.exists() and (potentialGateway / "modules" / "shared" / "configuration.py").exists(): + os.chdir(str(potentialGateway)) + import psycopg2 import psycopg2.extras +# Use the real APP_CONFIG which handles encryption and environment-specific configs +from modules.shared.configuration import APP_CONFIG # type: ignore + # Logging konfigurieren logging.basicConfig( level=logging.INFO, @@ -47,6 +74,27 @@ logging.basicConfig( ) logger = logging.getLogger(__name__) +# Force APP_CONFIG to refresh after changing working directory +# This ensures .env file is loaded from the correct location +try: + APP_CONFIG.refresh() + envType = APP_CONFIG.get('APP_ENV_TYPE', 'unknown') + keySysVar = APP_CONFIG.get('APP_KEY_SYSVAR', 'not_set') + logger.debug(f"APP_CONFIG refreshed. Environment type: {envType}") + logger.debug(f"APP_KEY_SYSVAR: {keySysVar}") + logger.debug(f"Current working directory: {os.getcwd()}") + + # Check if master key is available (needed for decrypting secrets) + if keySysVar != 'not_set': + masterKeyEnv = os.environ.get(keySysVar) + if masterKeyEnv: + logger.debug(f"Master key found in environment variable: {keySysVar}") + else: + logger.warning(f"Master key not found in environment variable: {keySysVar}") + logger.warning("Encrypted secrets may not be decryptable!") +except Exception as e: + logger.warning(f"Could not refresh APP_CONFIG: {e}") + # Alle PowerOn Datenbanken ALL_DATABASES = [ "poweron_app", # Haupt-App: User, Mandate, RBAC, Features @@ -56,65 +104,62 @@ ALL_DATABASES = [ "poweron_trustee", # Trustee ] - -def _loadEnvConfig() -> Dict[str, str]: - """Lädt die Konfiguration direkt aus der .env Datei.""" - config = {} - envPath = Path(__file__).parent / '.env' - - if not envPath.exists(): - logger.warning(f"Environment file not found at {envPath}") - return config - - # Versuche verschiedene Encodings - encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] - - for encoding in encodings: - try: - with open(envPath, 'r', encoding=encoding) as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' in line: - key, value = line.split('=', 1) - config[key.strip()] = value.strip() - # Erfolgreich geladen - return config - except UnicodeDecodeError: - continue - except Exception as e: - logger.error(f"Error loading .env file with {encoding}: {e}") - continue - - logger.error(f"Could not load .env file with any encoding") - return config - - -# Globale Konfiguration laden -_ENV_CONFIG = _loadEnvConfig() +# Datenbank-Konfiguration: Mapping von DB-Name zu Config-Prefix +# Jede Datenbank hat ihre eigenen Variablen: DB_APP_HOST, DB_CHAT_HOST, etc. +DATABASE_CONFIG = { + "poweron_app": "DB_APP", # DB_APP_HOST, DB_APP_USER, DB_APP_PASSWORD_SECRET, etc. + "poweron_chat": "DB_CHAT", # DB_CHAT_HOST, DB_CHAT_USER, etc. + "poweron_management": "DB_MANAGEMENT", + "poweron_realestate": "DB_REALESTATE", + "poweron_trustee": "DB_TRUSTEE", +} def _getConfigValue(key: str, default: str = None) -> str: - """Holt einen Konfigurationswert.""" - return _ENV_CONFIG.get(key, os.environ.get(key, default)) + """Holt einen Konfigurationswert über APP_CONFIG (unterstützt Verschlüsselung).""" + return APP_CONFIG.get(key, default) + + +def _getDbConfig(dbName: str) -> Dict[str, Any]: + """ + Holt die Datenbankverbindungs-Konfiguration für eine spezifische Datenbank. + Unterstützt sowohl DB-spezifische Variablen (DB_APP_HOST) als auch Fallback (DB_HOST). + """ + prefix = DATABASE_CONFIG.get(dbName, "DB") + + # Versuche zuerst DB-spezifische Variablen, dann Fallback auf allgemeine + host = _getConfigValue(f"{prefix}_HOST") or _getConfigValue("DB_HOST", "localhost") + user = _getConfigValue(f"{prefix}_USER") or _getConfigValue("DB_USER") + password = _getConfigValue(f"{prefix}_PASSWORD_SECRET") or _getConfigValue("DB_PASSWORD_SECRET") + port = _getConfigValue(f"{prefix}_PORT") or _getConfigValue("DB_PORT", "5432") + + return { + "host": host, + "user": user, + "password": password, + "port": int(port) if port else 5432 + } def _databaseExists(dbDatabase: str) -> bool: """Prüft ob eine Datenbank existiert.""" - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) + config = _getDbConfig(dbDatabase) + + if not config["user"]: + logger.warning(f"DB-User nicht gesetzt für Datenbank {dbDatabase}") + return False + if not config["password"]: + logger.warning(f"DB-Password nicht gesetzt für Datenbank {dbDatabase}") + return False try: # Verbinde zur postgres Datenbank um zu prüfen conn = psycopg2.connect( - host=dbHost, - port=dbPort, + host=config["host"], + port=config["port"], database="postgres", - user=dbUser, - password=dbPassword + user=config["user"], + password=config["password"] ) conn.autocommit = True @@ -128,37 +173,43 @@ def _databaseExists(dbDatabase: str) -> bool: conn.close() return exists except Exception as e: - logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase} auf {config['host']}:{config['port']}: {e}") return False def _getDbConnection(dbDatabase: str): """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" - # Erst prüfen ob Datenbank existiert - if not _databaseExists(dbDatabase): - logger.warning(f"Datenbank '{dbDatabase}' existiert nicht - übersprungen") + config = _getDbConfig(dbDatabase) + + # Prüfe ob wichtige Konfigurationswerte fehlen + if not config["user"]: + logger.error(f"DB-User nicht gesetzt für {dbDatabase} - kann keine Verbindung herstellen") + return None + if not config["password"]: + logger.error(f"DB-Password nicht gesetzt für {dbDatabase} - kann keine Verbindung herstellen") return None - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht auf {config['host']}:{config['port']} - übersprungen") + return None try: conn = psycopg2.connect( - host=dbHost, - port=dbPort, + host=config["host"], + port=config["port"], database=dbDatabase, - user=dbUser, - password=dbPassword, + user=config["user"], + password=config["password"], cursor_factory=psycopg2.extras.RealDictCursor ) # Autocommit muss VOR set_client_encoding gesetzt werden, um Transaction-Konflikte zu vermeiden conn.autocommit = True conn.set_client_encoding('UTF8') + logger.debug(f"Erfolgreich verbunden mit {config['host']}:{config['port']}/{dbDatabase}") return conn except Exception as e: - logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + logger.error(f"Datenbankverbindung zu {dbDatabase} auf {config['host']}:{config['port']} fehlgeschlagen: {e}") raise @@ -504,6 +555,23 @@ def exportDatabase( # Welche Datenbanken exportieren? databasesToExport = onlyDatabases if onlyDatabases else ALL_DATABASES + # Prüfe ob grundlegende Konfigurationswerte vorhanden sind + # APP_CONFIG.get() entschlüsselt automatisch Werte, die mit _SECRET enden + # Jede Datenbank hat ihre eigenen Variablen (DB_APP_USER, DB_CHAT_USER, etc.) + missingConfigs = [] + for dbName in databasesToExport: + config = _getDbConfig(dbName) + if not config["user"] or not config["password"]: + missingConfigs.append(f"{dbName}: User={'gesetzt' if config['user'] else 'FEHLT'}, Password={'gesetzt' if config['password'] else 'FEHLT'}") + + if missingConfigs: + logger.error("WICHTIG: Einige Datenbank-Konfigurationen fehlen!") + for missing in missingConfigs: + logger.error(f" {missing}") + logger.error(" Bitte Umgebungsvariablen setzen oder .env Datei konfigurieren") + logger.error(" Hinweis: Verschlüsselte Secrets benötigen APP_KEY_SYSVAR Umgebungsvariable!") + logger.error(" Erwartete Variablen: DB_APP_USER, DB_CHAT_USER, DB_MANAGEMENT_USER, etc.") + # Standard-Ausgabepfad generieren (im Log-Ordner) if not outputPath: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/tool_db_import_migration.py b/tool_db_import_migration.py deleted file mode 100644 index 1ab4e4fe..00000000 --- a/tool_db_import_migration.py +++ /dev/null @@ -1,612 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Datenbank Import-Tool für Migration. - -Dieses Script importiert Daten aus einer JSON-Migrationsdatei -in ALLE PowerOn PostgreSQL-Datenbanken. - -ACHTUNG: Dieses Script kann bestehende Daten überschreiben! -Bitte vor dem Import ein Backup erstellen. - -Datenbanken: - - poweron_app (User, Mandate, RBAC, Features, etc.) - - poweron_chat (Chat-Konversationen und Nachrichten) - - poweron_management (Workflows, Prompts, Connections, etc.) - - poweron_realestate (Real Estate Daten) - - poweron_trustee (Trustee Daten) - -Verwendung: - python tool_db_import_migration.py [--dry-run] [--force] - -Optionen: - --dry-run Simuliert den Import ohne Änderungen - --force Bestätigung überspringen - --clear-first Tabellen vor dem Import leeren - --only-tables Komma-getrennte Liste von Tabellen (nur diese importieren) - --only-db Komma-getrennte Liste von Datenbanken (nur diese importieren) -""" - -import os -import sys -import json -import argparse -import logging -import time -from datetime import datetime -from typing import Dict, List, Any, Optional -from pathlib import Path - -import psycopg2 -import psycopg2.extras - -# Logging konfigurieren -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S' -) -logger = logging.getLogger(__name__) - -# Alle PowerOn Datenbanken -ALL_DATABASES = [ - "poweron_app", - "poweron_chat", - "poweron_management", - "poweron_realestate", - "poweron_trustee", -] - - -def _loadEnvConfig() -> Dict[str, str]: - """Lädt die Konfiguration direkt aus der .env Datei.""" - config = {} - envPath = Path(__file__).parent / '.env' - - if not envPath.exists(): - logger.warning(f"Environment file not found at {envPath}") - return config - - # Versuche verschiedene Encodings - encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] - - for encoding in encodings: - try: - with open(envPath, 'r', encoding=encoding) as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' in line: - key, value = line.split('=', 1) - config[key.strip()] = value.strip() - # Erfolgreich geladen - return config - except UnicodeDecodeError: - continue - except Exception as e: - logger.error(f"Error loading .env file with {encoding}: {e}") - continue - - logger.error(f"Could not load .env file with any encoding") - return config - - -# Globale Konfiguration laden -_ENV_CONFIG = _loadEnvConfig() - - -def _getConfigValue(key: str, default: str = None) -> str: - """Holt einen Konfigurationswert.""" - return _ENV_CONFIG.get(key, os.environ.get(key, default)) - - -def _getUtcTimestamp() -> float: - """Gibt den aktuellen UTC-Timestamp zurück.""" - return time.time() - - -def _databaseExists(dbDatabase: str) -> bool: - """Prüft ob eine Datenbank existiert.""" - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) - - try: - conn = psycopg2.connect( - host=dbHost, - port=dbPort, - database="postgres", - user=dbUser, - password=dbPassword - ) - conn.autocommit = True - - with conn.cursor() as cursor: - cursor.execute( - "SELECT 1 FROM pg_database WHERE datname = %s", - (dbDatabase,) - ) - exists = cursor.fetchone() is not None - - conn.close() - return exists - except Exception as e: - logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") - return False - - -def _getDbConnection(dbDatabase: str, autocommit: bool = False): - """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" - # Erst prüfen ob Datenbank existiert - if not _databaseExists(dbDatabase): - logger.warning(f"Datenbank '{dbDatabase}' existiert nicht") - return None - - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) - - try: - conn = psycopg2.connect( - host=dbHost, - port=dbPort, - database=dbDatabase, - user=dbUser, - password=dbPassword, - cursor_factory=psycopg2.extras.RealDictCursor - ) - conn.set_client_encoding('UTF8') - conn.autocommit = autocommit - return conn - except Exception as e: - logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") - raise - - -def _getExistingTables(conn) -> List[str]: - """Gibt alle Tabellennamen in der Datenbank zurück.""" - with conn.cursor() as cursor: - cursor.execute(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_type = 'BASE TABLE' - ORDER BY table_name - """) - tables = [row["table_name"] for row in cursor.fetchall()] - return tables - - -def _getTableColumns(conn, tableName: str) -> List[str]: - """Gibt alle Spalten einer Tabelle zurück.""" - with conn.cursor() as cursor: - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = %s AND table_schema = 'public' - """, (tableName,)) - columns = [row["column_name"] for row in cursor.fetchall()] - return columns - - -def _clearTable(conn, tableName: str): - """Löscht alle Daten aus einer Tabelle.""" - with conn.cursor() as cursor: - cursor.execute(f'DELETE FROM "{tableName}"') - - -def _insertRecord(conn, tableName: str, record: Dict[str, Any], existingColumns: List[str]) -> bool: - """Fügt einen Datensatz in eine Tabelle ein (UPSERT).""" - filteredRecord = {k: v for k, v in record.items() if k in existingColumns} - - if not filteredRecord: - return False - - # Metadaten hinzufügen falls nicht vorhanden - currentTime = _getUtcTimestamp() - if "_createdAt" not in filteredRecord and "_createdAt" in existingColumns: - filteredRecord["_createdAt"] = currentTime - if "_modifiedAt" in existingColumns: - filteredRecord["_modifiedAt"] = currentTime - - columns = list(filteredRecord.keys()) - values = [] - - for col in columns: - value = filteredRecord[col] - if isinstance(value, (dict, list)): - values.append(json.dumps(value)) - else: - values.append(value) - - colNames = ", ".join([f'"{col}"' for col in columns]) - placeholders = ", ".join(["%s"] * len(columns)) - - updateCols = [col for col in columns if col not in ["id", "_createdAt", "_createdBy"]] - updateClause = ", ".join([f'"{col}" = EXCLUDED."{col}"' for col in updateCols]) - - if updateClause: - sql = f''' - INSERT INTO "{tableName}" ({colNames}) - VALUES ({placeholders}) - ON CONFLICT ("id") DO UPDATE SET {updateClause} - ''' - else: - sql = f''' - INSERT INTO "{tableName}" ({colNames}) - VALUES ({placeholders}) - ON CONFLICT ("id") DO NOTHING - ''' - - try: - with conn.cursor() as cursor: - cursor.execute(sql, values) - return True - except Exception as e: - logger.error(f"Fehler beim Einfügen in {tableName}: {e}") - return False - - -def loadMigrationFile(filePath: str) -> Dict[str, Any]: - """Lädt die Migrationsdatei.""" - logger.info(f"Lade Migrationsdatei: {filePath}") - - if not os.path.exists(filePath): - raise FileNotFoundError(f"Datei nicht gefunden: {filePath}") - - with open(filePath, "r", encoding="utf-8") as f: - data = json.load(f) - - # Validierung - unterstütze beide Formate (alt: tables, neu: databases) - if "databases" not in data and "tables" not in data: - raise ValueError("Ungültiges Migrationsformat: 'databases' oder 'tables' erforderlich") - - return data - - -def _importSingleDatabase( - dbName: str, - dbData: Dict[str, Any], - dryRun: bool, - clearFirst: bool, - onlyTables: Optional[List[str]] -) -> Dict[str, Any]: - """Importiert Daten in eine einzelne Datenbank.""" - stats = { - "imported": {}, - "skipped": {}, - "errors": {}, - "totalImported": 0, - "totalSkipped": 0, - "totalErrors": 0 - } - - conn = _getDbConnection(dbName) - if conn is None: - logger.warning(f" Datenbank '{dbName}' existiert nicht - übersprungen") - return stats - - try: - existingTables = _getExistingTables(conn) - tables = dbData.get("tables", {}) - - tablesToImport = list(tables.keys()) - if onlyTables: - tablesToImport = [t for t in tablesToImport if t in onlyTables] - - for tableName in tablesToImport: - records = tables[tableName] - - if tableName not in existingTables: - logger.warning(f" Tabelle '{tableName}' existiert nicht - übersprungen") - stats["skipped"][tableName] = len(records) - stats["totalSkipped"] += len(records) - continue - - if dryRun: - stats["imported"][tableName] = len(records) - stats["totalImported"] += len(records) - continue - - if clearFirst: - _clearTable(conn, tableName) - - existingColumns = _getTableColumns(conn, tableName) - - imported = 0 - errors = 0 - - for record in records: - if _insertRecord(conn, tableName, record, existingColumns): - imported += 1 - else: - errors += 1 - - stats["imported"][tableName] = imported - stats["totalImported"] += imported - - if errors > 0: - stats["errors"][tableName] = errors - stats["totalErrors"] += errors - - if imported > 0: - logger.info(f" {tableName}: {imported} importiert, {errors} Fehler") - - if not dryRun: - conn.commit() - else: - conn.rollback() - - return stats - - except Exception as e: - conn.rollback() - logger.error(f" Import fehlgeschlagen: {e}") - raise - - finally: - conn.close() - - -def importDatabase( - filePath: str, - dryRun: bool = False, - clearFirst: bool = False, - onlyTables: Optional[List[str]] = None, - onlyDatabases: Optional[List[str]] = None -) -> Dict[str, Any]: - """ - Importiert Daten aus einer Migrationsdatei. - - Args: - filePath: Pfad zur Migrationsdatei - dryRun: Nur simulieren - clearFirst: Tabellen vor Import leeren - onlyTables: Nur diese Tabellen importieren - onlyDatabases: Nur diese Datenbanken importieren - - Returns: - Import-Statistiken - """ - migrationData = loadMigrationFile(filePath) - meta = migrationData.get("meta", {}) - - logger.info(f"Migrationsdatei geladen:") - logger.info(f" Exportiert am: {meta.get('exportedAt', 'unbekannt')}") - logger.info(f" Quelle: {meta.get('exportedFrom', 'unbekannt')}") - - stats = { - "databases": {}, - "totalImported": 0, - "totalSkipped": 0, - "totalErrors": 0 - } - - # Neues Format (mehrere Datenbanken) - if "databases" in migrationData: - databases = migrationData["databases"] - logger.info(f" Datenbanken: {len(databases)}") - logger.info(f" Tabellen: {meta.get('totalTables', 'unbekannt')}") - logger.info(f" Datensätze: {meta.get('totalRecords', 'unbekannt')}") - - for dbName, dbData in databases.items(): - if onlyDatabases and dbName not in onlyDatabases: - continue - - logger.info(f"Importiere Datenbank: {dbName}") - dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) - - stats["databases"][dbName] = dbStats - stats["totalImported"] += dbStats["totalImported"] - stats["totalSkipped"] += dbStats["totalSkipped"] - stats["totalErrors"] += dbStats["totalErrors"] - - # Altes Format (einzelne Datenbank - poweron_app) - elif "tables" in migrationData: - logger.info(" Format: Legacy (einzelne Datenbank)") - dbName = "poweron_app" - dbData = {"tables": migrationData["tables"]} - - if not onlyDatabases or dbName in onlyDatabases: - logger.info(f"Importiere Datenbank: {dbName}") - dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) - - stats["databases"][dbName] = dbStats - stats["totalImported"] = dbStats["totalImported"] - stats["totalSkipped"] = dbStats["totalSkipped"] - stats["totalErrors"] = dbStats["totalErrors"] - - if dryRun: - logger.info("Dry-Run: Keine Änderungen vorgenommen") - - return stats - - -def printImportPreview(filePath: str): - """Zeigt eine Vorschau der zu importierenden Daten.""" - migrationData = loadMigrationFile(filePath) - meta = migrationData.get("meta", {}) - - print("\n" + "=" * 70) - print("IMPORT VORSCHAU") - print("=" * 70) - print(f"Datei: {filePath}") - print(f"Exportiert am: {meta.get('exportedAt', 'unbekannt')}") - print(f"Quelle: {meta.get('exportedFrom', 'unbekannt')}") - - # Neues Format - if "databases" in migrationData: - databases = migrationData["databases"] - print(f"Datenbanken: {len(databases)}") - print("=" * 70) - - grandTotal = 0 - for dbName, dbData in databases.items(): - tables = dbData.get("tables", {}) - dbTotal = sum(len(records) for records in tables.values()) - grandTotal += dbTotal - - print(f"\n{dbName} ({dbTotal} Datensätze)") - print("-" * 70) - print(f" {'Tabelle':<45} {'Datensätze':>15}") - print(f" {'-' * 45} {'-' * 15}") - - for tableName, records in sorted(tables.items()): - if len(records) > 0: - print(f" {tableName:<45} {len(records):>15}") - - print("\n" + "=" * 70) - print(f"GESAMT: {grandTotal} Datensätze") - - # Altes Format - elif "tables" in migrationData: - tables = migrationData["tables"] - print(f"Format: Legacy (poweron_app)") - print("-" * 70) - print(f"{'Tabelle':<45} {'Datensätze':>15}") - print("-" * 70) - - totalRecords = 0 - for tableName, records in sorted(tables.items()): - count = len(records) - totalRecords += count - if count > 0: - print(f"{tableName:<45} {count:>15}") - - print("-" * 70) - print(f"{'GESAMT':<45} {totalRecords:>15}") - - print("=" * 70 + "\n") - - -def main(): - parser = argparse.ArgumentParser( - description="Importiert Datenbank-Daten aus einer Migrationsdatei", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Datenbanken: - poweron_app - User, Mandate, RBAC, Features - poweron_chat - Chat-Konversationen - poweron_management - Workflows, Prompts, Connections - poweron_realestate - Real Estate Daten - poweron_trustee - Trustee Daten - -Beispiele: - python tool_db_import_migration.py migration_export.json --dry-run - python tool_db_import_migration.py migration_export.json --preview - python tool_db_import_migration.py migration_export.json --force - python tool_db_import_migration.py migration_export.json --clear-first --force - python tool_db_import_migration.py migration_export.json --only-db poweron_app - python tool_db_import_migration.py migration_export.json --only-tables UserInDB,Mandate - """ - ) - - parser.add_argument( - "import_file", - help="Pfad zur Migrationsdatei (JSON)", - type=str - ) - - parser.add_argument( - "--dry-run", - help="Simuliert den Import ohne Änderungen", - action="store_true" - ) - - parser.add_argument( - "--force", - help="Bestätigung überspringen", - action="store_true" - ) - - parser.add_argument( - "--clear-first", - help="Tabellen vor dem Import leeren", - action="store_true" - ) - - parser.add_argument( - "--only-tables", - help="Nur diese Tabellen importieren (komma-getrennt)", - type=str, - default="" - ) - - parser.add_argument( - "--only-db", - help="Nur diese Datenbank(en) importieren (komma-getrennt)", - type=str, - default="" - ) - - parser.add_argument( - "--preview", - help="Nur Vorschau anzeigen (kein Import)", - action="store_true" - ) - - args = parser.parse_args() - - # Nur Vorschau anzeigen - if args.preview: - printImportPreview(args.import_file) - return - - # Listen parsen - onlyTables = None - if args.only_tables: - onlyTables = [t.strip() for t in args.only_tables.split(",") if t.strip()] - - onlyDatabases = None - if args.only_db: - onlyDatabases = [db.strip() for db in args.only_db.split(",") if db.strip()] - - # Bestätigung einholen - if not args.dry_run and not args.force: - printImportPreview(args.import_file) - - if args.clear_first: - print("WARNUNG: --clear-first wird ALLE bestehenden Daten in den Zieltabellen löschen!") - - response = input("\nMöchten Sie den Import starten? [y/N]: ") - if response.lower() not in ["y", "yes", "j", "ja"]: - print("Import abgebrochen.") - return - - # Import durchführen - try: - if args.dry_run: - logger.info("=== DRY-RUN MODUS ===") - - stats = importDatabase( - filePath=args.import_file, - dryRun=args.dry_run, - clearFirst=args.clear_first, - onlyTables=onlyTables, - onlyDatabases=onlyDatabases - ) - - print("\n" + "=" * 70) - print("IMPORT ERGEBNIS") - print("=" * 70) - print(f"Importiert: {stats['totalImported']} Datensätze") - print(f"Übersprungen: {stats['totalSkipped']} Datensätze") - print(f"Fehler: {stats['totalErrors']} Datensätze") - - if args.dry_run: - print("\n(Dry-Run: Keine tatsächlichen Änderungen vorgenommen)") - else: - print("\n Import erfolgreich abgeschlossen!") - - print("=" * 70 + "\n") - - except Exception as e: - logger.error(f"Import fehlgeschlagen: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main() From 82c01b5cb009f2ec8a571b602ca79cd36d4f9097 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 00:32:47 +0100 Subject: [PATCH 05/32] saas mandates core done --- env_dev.env | 2 +- modules/datamodels/datamodelFeatures.py | 4 +- modules/datamodels/datamodelMembership.py | 24 +-- modules/datamodels/datamodelRbac.py | 12 +- modules/datamodels/datamodelUam.py | 14 +- modules/interfaces/interfaceBootstrap.py | 201 +++++++++++-------- modules/interfaces/interfaceDbAppObjects.py | 4 +- modules/interfaces/interfaceRbac.py | 8 + modules/routes/routeAdminRbacRoles.py | 32 +-- modules/routes/routeDataMandates.py | 43 ++-- modules/routes/routeDataUsers.py | 34 +++- modules/routes/routeFeatureWorkflow.py | 13 +- modules/routes/routeFeatures.py | 79 ++++---- modules/routes/routeRbac.py | 46 +++-- modules/routes/routeSecurityLocal.py | 79 ++++++-- modules/shared/attributeUtils.py | 56 ++++-- modules/shared/dbMultiTenantOptimizations.py | 42 +++- 17 files changed, 438 insertions(+), 255 deletions(-) diff --git a/env_dev.env b/env_dev.env index 93523018..ac5349a7 100644 --- a/env_dev.env +++ b/env_dev.env @@ -19,7 +19,7 @@ APP_JWT_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERjlrSktmZHVuQnJ1VVJDdndLaUcxZGJsT2Z APP_TOKEN_EXPIRY=300 # CORS Configuration -APP_ALLOWED_ORIGINS=http://localhost:8080,https://playground.poweron-center.net +APP_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5176,https://playground.poweron-center.net # Logging configuration APP_LOGGING_LOG_LEVEL = DEBUG diff --git a/modules/datamodels/datamodelFeatures.py b/modules/datamodels/datamodelFeatures.py index e772a19e..60153f28 100644 --- a/modules/datamodels/datamodelFeatures.py +++ b/modules/datamodels/datamodelFeatures.py @@ -6,6 +6,7 @@ import uuid from typing import Optional from pydantic import BaseModel, Field from modules.shared.attributeUtils import registerModelLabels +from modules.datamodels.datamodelUtils import TextMultilingual class Feature(BaseModel): @@ -17,8 +18,7 @@ class Feature(BaseModel): description="Unique feature code (Primary Key), z.B. 'trustee', 'chatbot'", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} ) - label: dict = Field( - default_factory=dict, + label: TextMultilingual = Field( description="Feature label in multiple languages (I18n)", json_schema_extra={"frontend_type": "multilingual", "frontend_readonly": False, "frontend_required": True} ) diff --git a/modules/datamodels/datamodelMembership.py b/modules/datamodels/datamodelMembership.py index e2cdb0b6..e100b23c 100644 --- a/modules/datamodels/datamodelMembership.py +++ b/modules/datamodels/datamodelMembership.py @@ -20,15 +20,15 @@ class UserMandate(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the user-mandate membership", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) userId: str = Field( description="FK → User.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/users/", "frontend_fk_display_field": "username"} ) mandateId: str = Field( description="FK → Mandate.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/mandates/", "frontend_fk_display_field": "name"} ) enabled: bool = Field( default=True, @@ -58,15 +58,15 @@ class FeatureAccess(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the feature access", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) userId: str = Field( description="FK → User.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/users/", "frontend_fk_display_field": "username"} ) featureInstanceId: str = Field( description="FK → FeatureInstance.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/feature-instances/", "frontend_fk_display_field": "name"} ) enabled: bool = Field( default=True, @@ -96,15 +96,15 @@ class UserMandateRole(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the junction record", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) userMandateId: str = Field( description="FK → UserMandate.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/user-mandates/", "frontend_fk_display_field": "userId"} ) roleId: str = Field( description="FK → Role.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/rbac/roles", "frontend_fk_display_field": "roleLabel"} ) @@ -127,15 +127,15 @@ class FeatureAccessRole(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the junction record", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) featureAccessId: str = Field( description="FK → FeatureAccess.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/feature-access/", "frontend_fk_display_field": "userId"} ) roleId: str = Field( description="FK → Role.id (CASCADE DELETE)", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_fk_source": "/api/rbac/roles", "frontend_fk_display_field": "roleLabel"} ) diff --git a/modules/datamodels/datamodelRbac.py b/modules/datamodels/datamodelRbac.py index 666470c8..64ad56a4 100644 --- a/modules/datamodels/datamodelRbac.py +++ b/modules/datamodels/datamodelRbac.py @@ -40,7 +40,7 @@ class Role(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the role", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) roleLabel: str = Field( description="Unique role label identifier (e.g., 'admin', 'user', 'viewer')", @@ -55,17 +55,17 @@ class Role(BaseModel): mandateId: Optional[str] = Field( default=None, description="FK → Mandate.id (CASCADE DELETE). Null = Global/Template role.", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_visible": True, "frontend_required": False, "frontend_fk_source": "/api/mandates/", "frontend_fk_display_field": "name"} ) featureInstanceId: Optional[str] = Field( default=None, description="FK → FeatureInstance.id (CASCADE DELETE). Null = Mandate-level or Global role.", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_visible": True, "frontend_required": False, "frontend_fk_source": "/api/feature-instances/", "frontend_fk_display_field": "name"} ) featureCode: Optional[str] = Field( default=None, description="Feature code (z.B. 'trustee') - für Template-Rollen", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) isSystemRole: bool = Field( @@ -100,11 +100,11 @@ class AccessRule(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the access rule", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) roleId: str = Field( description="FK → Role.id (CASCADE DELETE!)", - json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": True} + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": True, "frontend_fk_source": "/api/rbac/roles", "frontend_fk_display_field": "roleLabel"} ) context: AccessRuleContext = Field( description="Context type: DATA (database), UI (interface), RESOURCE (system resources). IMMUTABLE!", diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index f1c5da33..96a99fee 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -67,12 +67,17 @@ class Mandate(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the mandate", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) name: str = Field( description="Name of the mandate", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} ) + description: Optional[str] = Field( + default=None, + description="Description of the mandate", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False} + ) enabled: bool = Field( default=True, description="Indicates whether the mandate is enabled", @@ -86,6 +91,7 @@ registerModelLabels( { "id": {"en": "ID", "de": "ID", "fr": "ID"}, "name": {"en": "Name", "de": "Name", "fr": "Nom"}, + "description": {"en": "Description", "de": "Beschreibung", "fr": "Description"}, "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, }, ) @@ -143,11 +149,11 @@ class User(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the user", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False} ) username: str = Field( - description="Username for login", - json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} + description="Username for login (immutable after creation)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} ) email: Optional[EmailStr] = Field( default=None, diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 95b660ec..1d5f1139 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -30,6 +30,7 @@ from modules.datamodels.datamodelMembership import ( UserMandate, UserMandateRole, ) +from modules.datamodels.datamodelFeatures import Feature logger = logging.getLogger(__name__) @@ -55,6 +56,9 @@ def initBootstrap(db: DatabaseConnector) -> None: # Initialize roles FIRST (needed for AccessRules) initRoles(db) + # Initialize features (trustee, chatbot, etc.) + initFeatures(db) + # Initialize RBAC rules (uses roleIds from roles) initRbacRules(db) @@ -114,6 +118,13 @@ def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[s existingUsers = db.getRecordset(UserInDB, recordFilter={"username": "admin"}) if existingUsers: userId = existingUsers[0].get("id") + existingIsSysAdmin = existingUsers[0].get("isSysAdmin", False) + + # Ensure admin user has isSysAdmin=True + if not existingIsSysAdmin: + logger.info(f"Updating admin user {userId} to set isSysAdmin=True") + db.recordModify(UserInDB, userId, {"isSysAdmin": True}) + logger.info(f"Admin user already exists with ID {userId}") return userId @@ -175,6 +186,10 @@ def initRoles(db: DatabaseConnector) -> None: Initialize standard roles if they don't exist. Roles are created as GLOBAL (mandateId=None) template roles. + NOTE: SysAdmin is NOT a role - it's a flag (User.isSysAdmin). + SysAdmin users bypass RBAC entirely and have full system access. + These template roles are for mandate/feature-level access control. + Args: db: Database connector instance """ @@ -182,15 +197,9 @@ def initRoles(db: DatabaseConnector) -> None: global _roleIdCache _roleIdCache = {} + # Standard template roles for mandate/feature-level access + # NOTE: No "sysadmin" role - SysAdmin is a flag (User.isSysAdmin), not a role! standardRoles = [ - Role( - roleLabel="sysadmin", - description={"en": "System Administrator - Full access to all system resources", "de": "System-Administrator - Vollzugriff auf alle System-Ressourcen", "fr": "Administrateur système - Accès complet à toutes les ressources"}, - mandateId=None, # Global template role - featureInstanceId=None, - featureCode=None, - isSystemRole=True - ), Role( roleLabel="admin", description={"en": "Administrator - Manage users and resources within mandate scope", "de": "Administrator - Benutzer und Ressourcen im Mandanten verwalten", "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat"}, @@ -235,6 +244,63 @@ def initRoles(db: DatabaseConnector) -> None: logger.info("Roles initialization completed") +def initFeatures(db: DatabaseConnector) -> None: + """ + Initialize standard features if they don't exist. + + Features are global definitions that can be instantiated within mandates. + Each feature has a unique code (e.g., 'trustee', 'chatbot'). + + Args: + db: Database connector instance + """ + logger.info("Initializing features") + + # Standard features available in the system + standardFeatures = [ + Feature( + code="trustee", + label={"en": "Trustee", "de": "Treuhand", "fr": "Fiduciaire"}, + icon="mdi-briefcase" + ), + Feature( + code="chatbot", + label={"en": "Chatbot", "de": "Chatbot", "fr": "Chatbot"}, + icon="mdi-robot" + ), + Feature( + code="chatworkflow", + label={"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"}, + icon="mdi-message-cog" + ), + Feature( + code="neutralization", + label={"en": "Neutralization", "de": "Neutralisierung", "fr": "Neutralisation"}, + icon="mdi-shield-check" + ), + Feature( + code="realestate", + label={"en": "Real Estate", "de": "Immobilien", "fr": "Immobilier"}, + icon="mdi-home-city" + ), + ] + + existingFeatures = db.getRecordset(Feature) + existingCodes = {f.get("code") for f in existingFeatures} + + for feature in standardFeatures: + if feature.code not in existingCodes: + try: + db.recordCreate(Feature, feature.model_dump()) + logger.info(f"Created feature: {feature.code}") + except Exception as e: + logger.warning(f"Error creating feature {feature.code}: {e}") + else: + logger.debug(f"Feature {feature.code} already exists") + + logger.info("Features initialization completed") + + def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: """ Get role ID by label, using cache or database lookup. @@ -297,26 +363,15 @@ def _createDefaultRoleRules(db: DatabaseConnector) -> None: Create default role rules for generic access (item = null). Uses roleId instead of roleLabel. + NOTE: No rules for "sysadmin" - SysAdmin is a flag (User.isSysAdmin), not a role! + SysAdmin users bypass RBAC entirely via the isSysAdmin check in getRecordsetWithRBAC(). + Args: db: Database connector instance """ defaultRules = [] - # SysAdmin Role - Full access to all - sysadminId = _getRoleId(db, "sysadmin") - if sysadminId: - defaultRules.append(AccessRule( - roleId=sysadminId, - context=AccessRuleContext.DATA, - item=None, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) - - # Admin Role - Group-level access + # Admin Role - Group-level access (highest role-based permission) adminId = _getRoleId(db, "admin") if adminId: defaultRules.append(AccessRule( @@ -370,29 +425,21 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: These rules override generic rules for specific tables. Uses roleId instead of roleLabel. + NOTE: No rules for "sysadmin" - SysAdmin is a flag (User.isSysAdmin), not a role! + SysAdmin users bypass RBAC entirely via the isSysAdmin check in getRecordsetWithRBAC(). + Args: db: Database connector instance """ tableRules = [] - # Get role IDs - sysadminId = _getRoleId(db, "sysadmin") + # Get role IDs (no sysadmin - that's a flag, not a role!) adminId = _getRoleId(db, "admin") userId = _getRoleId(db, "user") viewerId = _getRoleId(db, "viewer") - # Mandate table - Only sysadmin can access - if sysadminId: - tableRules.append(AccessRule( - roleId=sysadminId, - context=AccessRuleContext.DATA, - item="Mandate", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) + # Mandate table - Only SysAdmin (flag) can access, not roles + # Regular roles have no access to Mandate table if adminId: tableRules.append(AccessRule( roleId=adminId, @@ -427,18 +474,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # UserInDB table - if sysadminId: - tableRules.append(AccessRule( - roleId=sysadminId, - context=AccessRuleContext.DATA, - item="UserInDB", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) + # UserInDB table - Admin can manage users within group scope if adminId: tableRules.append(AccessRule( roleId=adminId, @@ -474,6 +510,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: )) # Standard tables with typical access patterns + # NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag standardTables = [ "UserConnection", "DataNeutraliserConfig", "DataNeutralizerAttributes", "ChatWorkflow", "Prompt", "Projekt", "Parzelle", "Dokument", @@ -483,17 +520,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: ] for table in standardTables: - if sysadminId: - tableRules.append(AccessRule( - roleId=sysadminId, - context=AccessRuleContext.DATA, - item=table, - view=True, - read=AccessLevel.ALL, - create=AccessLevel.ALL, - update=AccessLevel.ALL, - delete=AccessLevel.ALL, - )) + # Admin gets full group-level access (highest role-based permission) if adminId: tableRules.append(AccessRule( roleId=adminId, @@ -528,28 +555,18 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # AuthEvent table - special handling - if sysadminId: - tableRules.append(AccessRule( - roleId=sysadminId, - context=AccessRuleContext.DATA, - item="AuthEvent", - view=True, - read=AccessLevel.ALL, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.ALL, - )) + # AuthEvent table - Audit logs (no delete allowed for audit integrity!) + # SysAdmin can delete via isSysAdmin bypass, but regular admins cannot if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, item="AuthEvent", view=True, - read=AccessLevel.ALL, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.ALL, + read=AccessLevel.ALL, # Admin can see all auth events for security monitoring + create=AccessLevel.NONE, # Events are system-generated + update=AccessLevel.NONE, # Audit logs are immutable + delete=AccessLevel.NONE, # NO delete - audit integrity! )) if userId: tableRules.append(AccessRule( @@ -557,7 +574,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: context=AccessRuleContext.DATA, item="AuthEvent", view=True, - read=AccessLevel.MY, + read=AccessLevel.MY, # Users can see their own auth events create=AccessLevel.NONE, update=AccessLevel.NONE, delete=AccessLevel.NONE, @@ -586,13 +603,15 @@ def _createUiContextRules(db: DatabaseConnector) -> None: Create UI context rules for controlling UI element visibility. Uses roleId instead of roleLabel. + NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag. + Args: db: Database connector instance """ uiRules = [] - # All roles get full UI access by default - for roleLabel in ["sysadmin", "admin", "user", "viewer"]: + # All roles get full UI access by default (no sysadmin - that's a flag) + for roleLabel in ["admin", "user", "viewer"]: roleId = _getRoleId(db, roleLabel) if roleId: uiRules.append(AccessRule( @@ -617,13 +636,15 @@ def _createResourceContextRules(db: DatabaseConnector) -> None: Create RESOURCE context rules for controlling resource access. Uses roleId instead of roleLabel. + NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag. + Args: db: Database connector instance """ resourceRules = [] - # All roles get full resource access by default - for roleLabel in ["sysadmin", "admin", "user", "viewer"]: + # All roles get full resource access by default (no sysadmin - that's a flag) + for roleLabel in ["admin", "user", "viewer"]: roleId = _getRoleId(db, roleLabel) if roleId: resourceRules.append(AccessRule( @@ -653,15 +674,19 @@ def assignInitialUserMemberships( Assign initial memberships to admin and event users via UserMandate + UserMandateRole. This is the NEW multi-tenant way of assigning roles. + NOTE: SysAdmin is a flag (User.isSysAdmin), not a role. Initial users get the "admin" role + within the root mandate, plus they have isSysAdmin=True for system-level access. + Args: db: Database connector instance mandateId: Root mandate ID adminUserId: Admin user ID eventUserId: Event user ID """ - sysadminRoleId = _getRoleId(db, "sysadmin") - if not sysadminRoleId: - logger.warning("Sysadmin role not found, skipping membership assignment") + # Use "admin" role for mandate membership (SysAdmin is a flag, not a role!) + adminRoleId = _getRoleId(db, "admin") + if not adminRoleId: + logger.warning("Admin role not found, skipping membership assignment") return for userId, userName in [(adminUserId, "admin"), (eventUserId, "event")]: @@ -688,17 +713,17 @@ def assignInitialUserMemberships( # Check if UserMandateRole already exists existingRoles = db.getRecordset( UserMandateRole, - recordFilter={"userMandateId": userMandateId, "roleId": sysadminRoleId} + recordFilter={"userMandateId": userMandateId, "roleId": adminRoleId} ) if not existingRoles: - # Create UserMandateRole + # Create UserMandateRole with "admin" role userMandateRole = UserMandateRole( userMandateId=userMandateId, - roleId=sysadminRoleId + roleId=adminRoleId ) db.recordCreate(UserMandateRole, userMandateRole) - logger.info(f"Assigned sysadmin role to {userName} user in mandate") + logger.info(f"Assigned admin role to {userName} user in mandate") def _getPasswordHash(password: Optional[str]) -> Optional[str]: diff --git a/modules/interfaces/interfaceDbAppObjects.py b/modules/interfaces/interfaceDbAppObjects.py index d661e603..ed8e1fc4 100644 --- a/modules/interfaces/interfaceDbAppObjects.py +++ b/modules/interfaces/interfaceDbAppObjects.py @@ -1313,13 +1313,13 @@ class AppObjects: return Mandate(**filteredMandates[0]) - def createMandate(self, name: str, language: str = "en") -> Mandate: + def createMandate(self, name: str, description: str = None, enabled: bool = True) -> Mandate: """Creates a new mandate if user has permission.""" if not self.checkRbacPermission(Mandate, "create"): raise PermissionError("No permission to create mandates") # Create mandate data using model - mandateData = Mandate(name=name, language=language) + mandateData = Mandate(name=name, description=description, enabled=enabled) # Create mandate record createdRecord = self.db.recordCreate(Mandate, mandateData) diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 26515e94..6b432a2b 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -59,6 +59,14 @@ def getRecordsetWithRBAC( if not connector._ensureTableExists(modelClass): return [] + # SysAdmin bypass: SysAdmin users have full access to all tables + isSysAdmin = getattr(currentUser, 'isSysAdmin', False) + if isSysAdmin: + logger.debug(f"SysAdmin user {currentUser.id} bypassing RBAC for table {table}") + # Direct access without RBAC filtering + # Note: getRecordset doesn't support orderBy/limit - these are only used in RBAC path + return connector.getRecordset(modelClass, recordFilter=recordFilter) + # Get RBAC permissions for this table # AccessRule table is always in DbApp database dbApp = getRootDbAppConnector() diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index a9397867..b7069f86 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Depends, Query, Body, Path, Reques from typing import List, Dict, Any, Optional, Set import logging -from modules.auth import limiter, requireSysAdmin, RequestContext +from modules.auth import limiter, requireSysAdmin from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelRbac import Role from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole @@ -78,7 +78,7 @@ router = APIRouter( @limiter.limit("60/minute") async def listRoles( request: Request, - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get list of all available roles with metadata. @@ -123,7 +123,7 @@ async def listRoles( @limiter.limit("60/minute") async def getRoleOptions( request: Request, - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get role options for select dropdowns. @@ -165,7 +165,7 @@ async def getRoleOptions( async def createRole( request: Request, role: Role = Body(...), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Create a new role. @@ -209,7 +209,7 @@ async def createRole( async def getRole( request: Request, roleId: str = Path(..., description="Role ID"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get a role by ID. @@ -254,7 +254,7 @@ async def updateRole( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update an existing role. @@ -301,7 +301,7 @@ async def updateRole( async def deleteRole( request: Request, roleId: str = Path(..., description="Role ID"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, str]: """ Delete a role. @@ -346,7 +346,7 @@ async def listUsersWithRoles( request: Request, roleLabel: Optional[str] = Query(None, description="Filter by role label"), mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get list of users with their role assignments. @@ -417,7 +417,7 @@ async def listUsersWithRoles( async def getUserRoles( request: Request, userId: str = Path(..., description="User ID"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get role assignments for a specific user. @@ -468,7 +468,7 @@ async def updateUserRoles( request: Request, userId: str = Path(..., description="User ID"), newRoleLabels: List[str] = Body(..., description="List of role labels to assign"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update role assignments for a specific user. @@ -535,7 +535,7 @@ async def updateUserRoles( newRole = UserMandateRole(userMandateId=userMandateId, roleId=roleId) interface.db.recordCreate(UserMandateRole, newRole.model_dump()) - logger.info(f"Updated roles for user {userId}: {newRoleLabels} by SysAdmin {context.user.id}") + logger.info(f"Updated roles for user {userId}: {newRoleLabels} by SysAdmin {currentUser.id}") userRoleLabels = _getUserRoleLabels(interface, userId) return { @@ -565,7 +565,7 @@ async def addUserRole( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to add"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Add a role to a user (if not already assigned). @@ -617,7 +617,7 @@ async def addUserRole( # Add the role newRole = UserMandateRole(userMandateId=userMandateId, roleId=str(role.id)) interface.db.recordCreate(UserMandateRole, newRole.model_dump()) - logger.info(f"Added role {roleLabel} to user {userId} by SysAdmin {context.user.id}") + logger.info(f"Added role {roleLabel} to user {userId} by SysAdmin {currentUser.id}") userRoleLabels = _getUserRoleLabels(interface, userId) return { @@ -647,7 +647,7 @@ async def removeUserRole( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to remove"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Remove a role from a user. @@ -697,7 +697,7 @@ async def removeUserRole( roleRemoved = True if roleRemoved: - logger.info(f"Removed role {roleLabel} from user {userId} by SysAdmin {context.user.id}") + logger.info(f"Removed role {roleLabel} from user {userId} by SysAdmin {currentUser.id}") userRoleLabels = _getUserRoleLabels(interface, userId) return { @@ -727,7 +727,7 @@ async def getUsersWithRole( request: Request, roleLabel: str = Path(..., description="Role label"), mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get all users with a specific role. diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 817ab762..ec109c6b 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -55,10 +55,10 @@ class MandateUserInfo(BaseModel): userId: str username: str email: Optional[str] - firstname: Optional[str] - lastname: Optional[str] + fullName: Optional[str] userMandateId: str roleIds: List[str] + roleLabels: List[str] # Resolved role labels for display enabled: bool # Configure logger @@ -79,7 +79,7 @@ router = APIRouter( async def get_mandates( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse[Mandate]: """ Get mandates with optional pagination, sorting, and filtering. @@ -143,7 +143,7 @@ async def get_mandates( async def get_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Mandate: """ Get a specific mandate by ID. @@ -174,7 +174,7 @@ async def get_mandate( async def create_mandate( request: Request, mandateData: dict = Body(..., description="Mandate data with at least 'name' field"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Mandate: """ Create a new mandate. @@ -192,14 +192,16 @@ async def create_mandate( ) # Get optional fields with defaults - language = mandateData.get('language', 'en') + description = mandateData.get('description') + enabled = mandateData.get('enabled', True) appInterface = interfaceDbAppObjects.getRootInterface() # Create mandate newMandate = appInterface.createMandate( name=name, - language=language + description=description, + enabled=enabled ) if not newMandate: @@ -208,7 +210,7 @@ async def create_mandate( detail="Failed to create mandate" ) - logger.info(f"Mandate {newMandate.id} created by SysAdmin {context.user.id}") + logger.info(f"Mandate {newMandate.id} created by SysAdmin {currentUser.id}") return newMandate except HTTPException: @@ -226,7 +228,7 @@ async def update_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate to update"), mandateData: dict = Body(..., description="Mandate update data"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Mandate: """ Update an existing mandate. @@ -254,7 +256,7 @@ async def update_mandate( detail="Failed to update mandate" ) - logger.info(f"Mandate {mandateId} updated by SysAdmin {context.user.id}") + logger.info(f"Mandate {mandateId} updated by SysAdmin {currentUser.id}") return updatedMandate except HTTPException: @@ -271,7 +273,7 @@ async def update_mandate( async def delete_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate to delete"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Delete a mandate. @@ -304,7 +306,7 @@ async def delete_mandate( detail=str(e) ) - logger.info(f"Mandate {mandateId} deleted by SysAdmin {context.user.id}") + logger.info(f"Mandate {mandateId} deleted by SysAdmin {currentUser.id}") return {"message": f"Mandate {mandateId} deleted successfully"} except HTTPException: @@ -360,21 +362,30 @@ async def listMandateUsers( result = [] for um in userMandates: # Get user info - user = rootInterface.getUserById(um.get("userId")) + user = rootInterface.getUser(um.get("userId")) if not user: continue # Get roles for this membership roleIds = rootInterface.getRoleIdsForUserMandate(um.get("id")) + # Resolve role labels for display + roleLabels = [] + for roleId in roleIds: + role = rootInterface.getRole(roleId) + if role: + roleLabels.append(role.roleLabel) + else: + roleLabels.append(roleId) # Fallback to ID if not found + result.append(MandateUserInfo( userId=str(user.id), username=user.username, email=user.email, - firstname=user.firstname, - lastname=user.lastname, + fullName=user.fullName, userMandateId=um.get("id"), roleIds=roleIds, + roleLabels=roleLabels, enabled=um.get("enabled", True) )) @@ -434,7 +445,7 @@ async def addUserToMandate( ) # 4. Verify target user exists - targetUser = rootInterface.getUserById(data.targetUserId) + targetUser = rootInterface.getUser(data.targetUserId) if not targetUser: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index a9ebbab5..81115e51 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -12,6 +12,7 @@ MULTI-TENANT: User management requires RequestContext. from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response, Query from typing import List, Dict, Any, Optional from fastapi import status +from pydantic import BaseModel import logging import json @@ -192,12 +193,22 @@ async def get_user( detail=f"Failed to get user: {str(e)}" ) +class CreateUserRequest(BaseModel): + """Request body for creating a new user""" + username: str + email: Optional[str] = None + fullName: Optional[str] = None + language: str = "en" + enabled: bool = True + isSysAdmin: bool = False + password: Optional[str] = None + + @router.post("", response_model=User) @limiter.limit("10/minute") async def create_user( request: Request, - user_data: User = Body(...), - password: Optional[str] = Body(None, embed=True), + userData: CreateUserRequest = Body(...), context: RequestContext = Depends(getRequestContext) ) -> User: """ @@ -206,16 +217,17 @@ async def create_user( """ appInterface = interfaceDbAppObjects.getInterface(context.user) - # Extract fields from User model and call createUser with individual parameters + # Extract fields from request model and call createUser with individual parameters from modules.datamodels.datamodelUam import AuthAuthority newUser = appInterface.createUser( - username=user_data.username, - password=password, - email=user_data.email, - fullName=user_data.fullName, - language=user_data.language, - enabled=user_data.enabled, - authenticationAuthority=user_data.authenticationAuthority + username=userData.username, + password=userData.password, + email=userData.email, + fullName=userData.fullName, + language=userData.language, + enabled=userData.enabled, + authenticationAuthority=AuthAuthority.LOCAL, + isSysAdmin=userData.isSysAdmin ) # MULTI-TENANT: Add user to current mandate via UserMandate with default "user" role @@ -303,7 +315,7 @@ async def reset_user_password( appInterface = interfaceDbAppObjects.getInterface(context.user) # Get target user - target_user = appInterface.getUserById(userId) + target_user = appInterface.getUser(userId) if not target_user: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/modules/routes/routeFeatureWorkflow.py b/modules/routes/routeFeatureWorkflow.py index ac002332..33b9c0fc 100644 --- a/modules/routes/routeFeatureWorkflow.py +++ b/modules/routes/routeFeatureWorkflow.py @@ -13,6 +13,7 @@ import logging # Import interfaces and models import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext +from modules.datamodels.datamodelUam import User # Configure logger logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ router = APIRouter( @limiter.limit("30/minute") async def get_all_automation_events( request: Request, - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get all automation events across all mandates (sysadmin only). @@ -68,7 +69,7 @@ async def get_all_automation_events( @limiter.limit("5/minute") async def sync_all_automation_events( request: Request, - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Manually trigger sync for all automations (sysadmin only). @@ -79,7 +80,7 @@ async def sync_all_automation_events( from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.features.workflow import syncAutomationEvents - chatInterface = getChatInterface(context.user) + chatInterface = getChatInterface(currentUser) # Get event user for sync operation (routes can import from interfaces) rootInterface = getRootInterface() eventUser = rootInterface.getUserByUsername("event") @@ -90,7 +91,7 @@ async def sync_all_automation_events( ) from modules.services import getInterface as getServices - services = getServices(context.user, None) + services = getServices(currentUser, None) result = await syncAutomationEvents(services, eventUser) return { "success": True, @@ -111,7 +112,7 @@ async def sync_all_automation_events( async def remove_event( request: Request, eventId: str = Path(..., description="Event ID to remove"), - context: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Manually remove a specific event from scheduler (sysadmin only). @@ -126,7 +127,7 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatbot.getInterface(context.user) + chatInterface = interfaceDbChatbot.getInterface(currentUser) automation = chatInterface.getAutomationDefinition(automation_id) if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 01e33eac..326adeb7 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -295,42 +295,6 @@ def _mergeAccessLevel(current: str, new: str) -> str: return current -@router.get("/{featureCode}", response_model=Dict[str, Any]) -@limiter.limit("60/minute") -async def getFeature( - request: Request, - featureCode: str, - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """ - Get a specific feature by code. - - Args: - featureCode: Feature code (e.g., 'trustee', 'chatbot') - """ - try: - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - - feature = featureInterface.getFeature(featureCode) - if not feature: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Feature '{featureCode}' not found" - ) - - return feature.model_dump() - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting feature {featureCode}: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get feature: {str(e)}" - ) - - @router.post("/", response_model=Dict[str, Any]) @limiter.limit("10/minute") async def createFeature( @@ -745,6 +709,49 @@ async def createTemplateRole( ) +# ============================================================================= +# Dynamic Feature Route (MUST be last to avoid catching /instances, /my, etc.) +# ============================================================================= + +@router.get("/{featureCode}", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getFeature( + request: Request, + featureCode: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Get a specific feature by code. + + IMPORTANT: This route must be defined LAST to avoid catching paths like + /instances, /my, /templates, etc. + + Args: + featureCode: Feature code (e.g., 'trustee', 'chatbot') + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + feature = featureInterface.getFeature(featureCode) + if not feature: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature '{featureCode}' not found" + ) + + return feature.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting feature {featureCode}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature: {str(e)}" + ) + + # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeRbac.py b/modules/routes/routeRbac.py index d2e92c82..03c66ae0 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeRbac.py @@ -230,7 +230,7 @@ async def getAccessRules( context: Optional[str] = Query(None, description="Filter by context (DATA, UI, RESOURCE)"), item: Optional[str] = Query(None, description="Filter by item identifier"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse: """ Get access rules with optional filters. @@ -316,7 +316,7 @@ async def getAccessRules( async def getAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> dict: """ Get a specific access rule by ID. @@ -358,7 +358,7 @@ async def getAccessRule( async def createAccessRule( request: Request, accessRuleData: dict = Body(..., description="Access rule data"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> dict: """ Create a new access rule. @@ -404,7 +404,7 @@ async def createAccessRule( # Create rule createdRule = interface.createAccessRule(accessRule) - logger.info(f"Created access rule {createdRule.id} by SysAdmin {reqContext.user.id}") + logger.info(f"Created access rule {createdRule.id} by SysAdmin {currentUser.id}") # Convert to dict for JSON serialization return createdRule.model_dump() @@ -425,7 +425,7 @@ async def updateAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), accessRuleData: dict = Body(..., description="Updated access rule data"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> dict: """ Update an existing access rule. @@ -487,7 +487,7 @@ async def updateAccessRule( # Update rule updatedRule = interface.updateAccessRule(ruleId, accessRule) - logger.info(f"Updated access rule {ruleId} by SysAdmin {reqContext.user.id}") + logger.info(f"Updated access rule {ruleId} by SysAdmin {currentUser.id}") # Convert to dict for JSON serialization return updatedRule.model_dump() @@ -507,7 +507,7 @@ async def updateAccessRule( async def deleteAccessRule( request: Request, ruleId: str = Path(..., description="Access rule ID"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> dict: """ Delete an access rule. @@ -540,7 +540,7 @@ async def deleteAccessRule( detail=f"Failed to delete access rule {ruleId}" ) - logger.info(f"Deleted access rule {ruleId} by SysAdmin {reqContext.user.id}") + logger.info(f"Deleted access rule {ruleId} by SysAdmin {currentUser.id}") return {"success": True, "message": f"Access rule {ruleId} deleted successfully"} @@ -565,7 +565,7 @@ async def deleteAccessRule( async def listRoles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse: """ Get list of all available roles with metadata. @@ -604,6 +604,9 @@ async def listRoles( "id": role.id, "roleLabel": role.roleLabel, "description": role.description, + "mandateId": role.mandateId, + "featureInstanceId": role.featureInstanceId, + "featureCode": role.featureCode, "userCount": roleCounts.get(str(role.id), 0), "isSystemRole": role.isSystemRole }) @@ -662,7 +665,7 @@ async def listRoles( @limiter.limit("60/minute") async def getRoleOptions( request: Request, - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: """ Get role options for select dropdowns. @@ -704,7 +707,7 @@ async def getRoleOptions( async def createRole( request: Request, role: Role = Body(...), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Create a new role. @@ -721,12 +724,15 @@ async def createRole( createdRole = interface.createRole(role) - logger.info(f"Created role {createdRole.roleLabel} by SysAdmin {reqContext.user.id}") + logger.info(f"Created role {createdRole.roleLabel} by SysAdmin {currentUser.id}") return { "id": createdRole.id, "roleLabel": createdRole.roleLabel, "description": createdRole.description, + "mandateId": createdRole.mandateId, + "featureInstanceId": createdRole.featureInstanceId, + "featureCode": createdRole.featureCode, "isSystemRole": createdRole.isSystemRole } @@ -750,7 +756,7 @@ async def createRole( async def getRole( request: Request, roleId: str = Path(..., description="Role ID"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Get a role by ID. @@ -776,6 +782,9 @@ async def getRole( "id": role.id, "roleLabel": role.roleLabel, "description": role.description, + "mandateId": role.mandateId, + "featureInstanceId": role.featureInstanceId, + "featureCode": role.featureCode, "isSystemRole": role.isSystemRole } @@ -795,7 +804,7 @@ async def updateRole( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, Any]: """ Update an existing role. @@ -815,12 +824,15 @@ async def updateRole( updatedRole = interface.updateRole(roleId, role) - logger.info(f"Updated role {roleId} by SysAdmin {reqContext.user.id}") + logger.info(f"Updated role {roleId} by SysAdmin {currentUser.id}") return { "id": updatedRole.id, "roleLabel": updatedRole.roleLabel, "description": updatedRole.description, + "mandateId": updatedRole.mandateId, + "featureInstanceId": updatedRole.featureInstanceId, + "featureCode": updatedRole.featureCode, "isSystemRole": updatedRole.isSystemRole } @@ -844,7 +856,7 @@ async def updateRole( async def deleteRole( request: Request, roleId: str = Path(..., description="Role ID"), - reqContext: RequestContext = Depends(requireSysAdmin) + currentUser: User = Depends(requireSysAdmin) ) -> Dict[str, str]: """ Delete a role. @@ -866,7 +878,7 @@ async def deleteRole( detail=f"Role {roleId} not found" ) - logger.info(f"Deleted role {roleId} by SysAdmin {reqContext.user.id}") + logger.info(f"Deleted role {roleId} by SysAdmin {currentUser.id}") return {"message": f"Role {roleId} deleted successfully"} diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 8589db04..96f22136 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -24,6 +24,55 @@ from modules.shared.configuration import APP_CONFIG # Configure logger logger = logging.getLogger(__name__) + +def _sendAuthEmail(recipient: str, subject: str, message: str, userId: str = None) -> bool: + """ + Send authentication-related email directly without requiring full Services initialization. + Used for registration, password reset, and other auth flows. + + Args: + recipient: Email address + subject: Email subject + message: Plain text message (will be converted to HTML) + userId: Optional user ID for logging + + Returns: + bool: True if email was sent successfully + """ + try: + import html + from modules.interfaces.interfaceMessaging import getInterface as getMessagingInterface + from modules.datamodels.datamodelMessaging import MessagingChannel + + # Convert plain text to simple HTML + escaped = html.escape(message) + escaped = escaped.replace('\n', '
\n') + htmlMessage = f""" + + + +{escaped} + +""" + + messagingInterface = getMessagingInterface() + success = messagingInterface.send( + channel=MessagingChannel.EMAIL, + recipient=recipient, + subject=subject, + message=htmlMessage + ) + + if success: + logger.info(f"Auth email sent successfully to {recipient} (userId: {userId})") + else: + logger.warning(f"Failed to send auth email to {recipient} (userId: {userId})") + + return success + except Exception as e: + logger.error(f"Error sending auth email to {recipient}: {str(e)}", exc_info=True) + return False + # Create router for Local Security endpoints router = APIRouter( prefix="/api/local", @@ -261,15 +310,11 @@ async def register_user( # Send registration email with magic link try: - from modules.services import Services - services = Services(user) - magicLink = f"{baseUrl}/reset?token={token}" expiryHours = int(APP_CONFIG.get("Auth_RESET_TOKEN_EXPIRY_HOURS", "24")) emailSubject = "PowerOn Registrierung - Passwort setzen" - emailBody = f""" -Hallo {user.fullName or user.username}, + emailBody = f"""Hallo {user.fullName or user.username}, Vielen Dank für Ihre Registrierung bei PowerOn. @@ -280,10 +325,9 @@ Klicken Sie auf den folgenden Link, um Ihr Passwort zu setzen: Dieser Link ist {expiryHours} Stunden gültig. -Falls Sie sich nicht registriert haben, können Sie diese E-Mail ignorieren. -""" +Falls Sie sich nicht registriert haben, können Sie diese E-Mail ignorieren.""" - emailSent = services.messaging.sendEmailDirect( + emailSent = _sendAuthEmail( recipient=user.email, subject=emailSubject, message=emailBody, @@ -529,7 +573,6 @@ async def passwordResetRequest( user = rootInterface.findUserByUsernameLocalAuth(username) if user and user.email: - from modules.services import Services expiryHours = int(APP_CONFIG.get("Auth_RESET_TOKEN_EXPIRY_HOURS", "24")) try: @@ -539,16 +582,12 @@ async def passwordResetRequest( # Set reset token (clears password) rootInterface.setResetToken(user.id, token, expires) - # Get services for email sending - services = Services(user) - # Generate magic link using provided frontend URL magicLink = f"{baseUrl}/reset?token={token}" - # Send email + # Send email using dedicated auth email function emailSubject = "PowerOn - Passwort zurücksetzen" - emailBody = f""" -Hallo {user.fullName or user.username}, + emailBody = f"""Hallo {user.fullName or user.username}, Sie haben eine Passwort-Zurücksetzung für Ihren PowerOn Account angefordert. @@ -559,17 +598,19 @@ Klicken Sie auf den folgenden Link, um Ihr Passwort zurückzusetzen: Dieser Link ist {expiryHours} Stunden gültig. -Falls Sie diese Anforderung nicht gestellt haben, können Sie diese E-Mail ignorieren. -""" +Falls Sie diese Anforderung nicht gestellt haben, können Sie diese E-Mail ignorieren.""" - services.messaging.sendEmailDirect( + emailSent = _sendAuthEmail( recipient=user.email, subject=emailSubject, message=emailBody, userId=str(user.id) ) - logger.info(f"Password reset email sent to {user.email} for user {user.username}") + if emailSent: + logger.info(f"Password reset email sent to {user.email} for user {user.username}") + else: + logger.warning(f"Failed to send password reset email to {user.email}") except Exception as userErr: logger.error(f"Failed to send reset email for user {username}: {str(userErr)}") else: diff --git a/modules/shared/attributeUtils.py b/modules/shared/attributeUtils.py index 8f8804ee..5f6c2531 100644 --- a/modules/shared/attributeUtils.py +++ b/modules/shared/attributeUtils.py @@ -120,6 +120,9 @@ def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguag frontend_readonly = False frontend_required = field.is_required() frontend_options = None + frontend_visible = True # Default visible + frontend_fk_source = None # FK dropdown source (e.g., "/api/users/") + frontend_fk_display_field = None # Which field of the FK target to display (e.g., "username", "name") if field_info: # Try direct attributes first (though these won't exist for custom kwargs) @@ -167,6 +170,15 @@ def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguag frontend_required = json_extra.get("frontend_required", frontend_required) if frontend_options is None and "frontend_options" in json_extra: frontend_options = json_extra.get("frontend_options") + # Extract frontend_visible (default True, can be set to False to hide field) + if "frontend_visible" in json_extra: + frontend_visible = json_extra.get("frontend_visible", True) + # Extract frontend_fk_source for FK dropdown references + if "frontend_fk_source" in json_extra: + frontend_fk_source = json_extra.get("frontend_fk_source") + # Extract frontend_fk_display_field - which field of FK target to display + if "frontend_fk_display_field" in json_extra: + frontend_fk_display_field = json_extra.get("frontend_fk_display_field") # Use frontend type if available, otherwise detect from Python type if frontend_type: @@ -215,22 +227,34 @@ def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguag except Exception: pass - attributes.append( - { - "name": name, - "type": field_type, - "required": frontend_required, - "description": description, - "label": labels.get(name, name), - "placeholder": f"Please enter {labels.get(name, name)}", - "editable": not frontend_readonly, - "visible": True, - "order": len(attributes), - "readonly": frontend_readonly, - "options": frontend_options, - "default": field_default, - } - ) + # Hide "id" fields by default unless explicitly set to visible + # Also hide fields ending with "Id" that are FK references (unless they have fkSource) + if name == "id": + frontend_visible = False # Never show primary key in forms/tables + + attr_def = { + "name": name, + "type": field_type, + "required": frontend_required, + "description": description, + "label": labels.get(name, name), + "placeholder": f"Please enter {labels.get(name, name)}", + "editable": not frontend_readonly, + "visible": frontend_visible, + "order": len(attributes), + "readonly": frontend_readonly, + "options": frontend_options, + "default": field_default, + } + + # Add FK source for dropdown rendering if specified + if frontend_fk_source: + attr_def["fkSource"] = frontend_fk_source + # Also add display field if specified (which field of FK target to show) + if frontend_fk_display_field: + attr_def["fkDisplayField"] = frontend_fk_display_field + + attributes.append(attr_def) return {"model": model_label, "attributes": attributes} diff --git a/modules/shared/dbMultiTenantOptimizations.py b/modules/shared/dbMultiTenantOptimizations.py index 3e056179..ff8d7e8f 100644 --- a/modules/shared/dbMultiTenantOptimizations.py +++ b/modules/shared/dbMultiTenantOptimizations.py @@ -96,7 +96,7 @@ _PARTIAL_INDEXES = [ _FOREIGN_KEYS = [ # UserMandate FKs ("UserMandate", "fk_usermandate_mandate", "mandateId", "Mandate", "id"), - ("UserMandate", "fk_usermandate_user", "userId", "User", "id"), + ("UserMandate", "fk_usermandate_user", "userId", "UserInDB", "id"), # FeatureInstance FKs ("FeatureInstance", "fk_featureinstance_mandate", "mandateId", "Mandate", "id"), @@ -107,7 +107,7 @@ _FOREIGN_KEYS = [ # FeatureAccess FKs ("FeatureAccess", "fk_featureaccess_instance", "featureInstanceId", "FeatureInstance", "id"), - ("FeatureAccess", "fk_featureaccess_user", "userId", "User", "id"), + ("FeatureAccess", "fk_featureaccess_user", "userId", "UserInDB", "id"), # AccessRule FKs ("AccessRule", "fk_accessrule_role", "roleId", "Role", "id"), @@ -133,6 +133,9 @@ _IMMUTABLE_TRIGGERS = [ # AccessRule: context, roleId are immutable ("AccessRule", "tr_accessrule_immutable", ["context", "roleId"]), + + # User: username is immutable (login name cannot be changed) + ("UserInDB", "tr_user_immutable", ["username"]), ] @@ -340,6 +343,25 @@ def _applyIndexes(cursor, tables: Optional[List[str]]) -> int: return created +def _getForeignKeyTarget(cursor, constraintName: str) -> Optional[str]: + """Get the target table of an existing FK constraint.""" + cursor.execute(""" + SELECT ccu.table_name AS foreign_table_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.constraint_name = %s + LIMIT 1 + """, (constraintName,)) + row = cursor.fetchone() + if row: + if isinstance(row, dict): + return row.get('foreign_table_name') + return row[0] + return None + + def _applyForeignKeys(cursor, tables: Optional[List[str]]) -> int: """Apply foreign key constraints with CASCADE DELETE. Returns count created.""" created = 0 @@ -351,8 +373,22 @@ def _applyForeignKeys(cursor, tables: Optional[List[str]]) -> int: continue if not _tableExists(cursor, refTable): continue + + # Check if constraint exists if _constraintExists(cursor, constraintName): - continue + # Verify it points to the correct table + currentTarget = _getForeignKeyTarget(cursor, constraintName) + if currentTarget == refTable: + # FK exists and points to correct table - skip + continue + else: + # FK exists but points to wrong table - drop and recreate + logger.info(f"FK {constraintName} points to {currentTarget}, expected {refTable} - recreating") + try: + cursor.execute(f'ALTER TABLE "{tableName}" DROP CONSTRAINT "{constraintName}"') + except Exception as e: + logger.warning(f"Failed to drop FK {constraintName}: {e}") + continue try: cursor.execute(f""" From f3b01c823e25728b3f53390c52798a810a279a69 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 10:34:42 +0100 Subject: [PATCH 06/32] saas multi-mandate rbac up and running --- modules/connectors/connectorDbPostgre.py | 8 +- modules/interfaces/interfaceBootstrap.py | 170 +++++++ modules/routes/routeFeatures.py | 463 ++++++++++++++++++ modules/routes/routeRbac.py | 20 +- .../script_db_adapt_to_models.py | 4 +- scripts/script_db_cleanup_duplicate_roles.py | 189 +++++++ .../script_db_export_migration.py | 27 +- .../script_security_encrypt_all_env_files.py | 25 +- .../script_security_encrypt_config_value.py | 21 +- .../script_security_generate_master_keys.py | 12 +- .../script_stats_durations_from_log.py | 0 .../script_stats_get_codelines.py | 0 .../script_stats_showUnusedFunctions.py | 11 +- 13 files changed, 903 insertions(+), 47 deletions(-) rename tool_db_adapt_to_models.py => scripts/script_db_adapt_to_models.py (99%) create mode 100644 scripts/script_db_cleanup_duplicate_roles.py rename tool_db_export_migration.py => scripts/script_db_export_migration.py (97%) rename tool_security_encrypt_all_env_files.py => scripts/script_security_encrypt_all_env_files.py (95%) rename tool_security_encrypt_config_value.py => scripts/script_security_encrypt_config_value.py (96%) rename tool_security_generate_master_keys.py => scripts/script_security_generate_master_keys.py (88%) rename tool_stats_durations_from_log.py => scripts/script_stats_durations_from_log.py (100%) rename tool_stats_get_codelines.py => scripts/script_stats_get_codelines.py (100%) rename tool_stats_showUnusedFunctions.py => scripts/script_stats_showUnusedFunctions.py (96%) diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 5cf2dc62..2dfec2b4 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -853,8 +853,12 @@ class DatabaseConnector: if recordFilter: for field, value in recordFilter.items(): - where_conditions.append(f'"{field}" = %s') - where_values.append(value) + if value is None: + # Use IS NULL for None values (= NULL is always false in SQL) + where_conditions.append(f'"{field}" IS NULL') + else: + where_conditions.append(f'"{field}" = %s') + where_values.append(value) # Build the query if where_conditions: diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 1d5f1139..545b0040 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -298,9 +298,179 @@ def initFeatures(db: DatabaseConnector) -> None: else: logger.debug(f"Feature {feature.code} already exists") + # Initialize feature-specific template roles + _initFeatureTemplateRoles(db) + logger.info("Features initialization completed") +def _initFeatureTemplateRoles(db: DatabaseConnector) -> None: + """ + Initialize feature-specific template roles. + + These are global template roles (mandateId=None, featureInstanceId=None) + that get copied when a new FeatureInstance is created. + + Template roles are NOT system roles (isSystemRole=False) and can be + modified or deleted by administrators. + + Args: + db: Database connector instance + """ + logger.info("Initializing feature-specific template roles") + + # Feature-specific template roles definition + # Each feature has its own set of roles with appropriate descriptions + featureTemplateRoles = { + "trustee": [ + { + "roleLabel": "trustee-admin", + "description": { + "en": "Trustee Administrator - Full access to all trustee data and settings", + "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", + "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" + } + }, + { + "roleLabel": "trustee-accountant", + "description": { + "en": "Trustee Accountant - Manage accounting and financial data", + "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", + "fr": "Comptable fiduciaire - Gérer les données comptables et financières" + } + }, + { + "roleLabel": "trustee-client", + "description": { + "en": "Trustee Client - View own accounting data and documents", + "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", + "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" + } + }, + ], + "chatbot": [ + { + "roleLabel": "chatbot-admin", + "description": { + "en": "Chatbot Administrator - Full access to chatbot settings and all conversations", + "de": "Chatbot-Administrator - Vollzugriff auf Chatbot-Einstellungen und alle Konversationen", + "fr": "Administrateur chatbot - Accès complet aux paramètres et conversations" + } + }, + { + "roleLabel": "chatbot-user", + "description": { + "en": "Chatbot User - Use chatbot and view own conversations", + "de": "Chatbot-Benutzer - Chatbot nutzen und eigene Konversationen einsehen", + "fr": "Utilisateur chatbot - Utiliser le chatbot et consulter ses conversations" + } + }, + ], + "chatworkflow": [ + { + "roleLabel": "workflow-admin", + "description": { + "en": "Workflow Administrator - Full access to workflow configuration and execution", + "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", + "fr": "Administrateur workflow - Accès complet à la configuration et exécution" + } + }, + { + "roleLabel": "workflow-editor", + "description": { + "en": "Workflow Editor - Create and modify workflows", + "de": "Workflow-Editor - Workflows erstellen und bearbeiten", + "fr": "Éditeur workflow - Créer et modifier les workflows" + } + }, + { + "roleLabel": "workflow-viewer", + "description": { + "en": "Workflow Viewer - View workflows and execution results", + "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", + "fr": "Visualiseur workflow - Consulter les workflows et résultats" + } + }, + ], + "neutralization": [ + { + "roleLabel": "neutralization-admin", + "description": { + "en": "Neutralization Administrator - Full access to neutralization settings and data", + "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", + "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" + } + }, + { + "roleLabel": "neutralization-analyst", + "description": { + "en": "Neutralization Analyst - Analyze and process neutralization data", + "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", + "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" + } + }, + ], + "realestate": [ + { + "roleLabel": "realestate-admin", + "description": { + "en": "Real Estate Administrator - Full access to all property data and settings", + "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", + "fr": "Administrateur immobilier - Accès complet aux données et paramètres" + } + }, + { + "roleLabel": "realestate-manager", + "description": { + "en": "Real Estate Manager - Manage properties and tenants", + "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", + "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" + } + }, + { + "roleLabel": "realestate-viewer", + "description": { + "en": "Real Estate Viewer - View property information", + "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", + "fr": "Visualiseur immobilier - Consulter les informations immobilières" + } + }, + ], + } + + # Get existing template roles (mandateId=None, featureCode set) + existingRoles = db.getRecordset(Role, recordFilter={"mandateId": None}) + existingRoleKeys = { + (r.get("featureCode"), r.get("roleLabel")) + for r in existingRoles + if r.get("featureCode") is not None + } + + createdCount = 0 + for featureCode, roles in featureTemplateRoles.items(): + for roleDef in roles: + roleKey = (featureCode, roleDef["roleLabel"]) + if roleKey not in existingRoleKeys: + try: + templateRole = Role( + roleLabel=roleDef["roleLabel"], + description=roleDef["description"], + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=featureCode, + isSystemRole=False # Can be deleted by admins + ) + db.recordCreate(Role, templateRole) + createdCount += 1 + logger.info(f"Created template role: {roleDef['roleLabel']} for feature {featureCode}") + except Exception as e: + logger.warning(f"Error creating template role {roleDef['roleLabel']}: {e}") + else: + logger.debug(f"Template role {roleDef['roleLabel']} for {featureCode} already exists") + + logger.info(f"Feature template roles initialization completed ({createdCount} created)") + + def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: """ Get role ID by label, using cache or database lookup. diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 326adeb7..a9a124a8 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -709,6 +709,469 @@ async def createTemplateRole( ) +# ============================================================================= +# Feature Instance Users Endpoints +# Manage which users have access to a specific feature instance +# ============================================================================= + +class FeatureInstanceUserCreate(BaseModel): + """Request model for adding a user to a feature instance""" + userId: str = Field(..., description="User ID to add") + roleIds: List[str] = Field(default_factory=list, description="Role IDs to assign") + + +class FeatureInstanceUserResponse(BaseModel): + """Response model for a user in a feature instance""" + userId: str + username: str + email: Optional[str] + fullName: Optional[str] + featureAccessId: str + roleIds: List[str] + roleLabels: List[str] + enabled: bool + + +@router.get("/instances/{instanceId}/users", response_model=List[FeatureInstanceUserResponse]) +@limiter.limit("60/minute") +async def listFeatureInstanceUsers( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> List[FeatureInstanceUserResponse]: + """ + List all users with access to a specific feature instance. + + Returns users and their roles for the given instance. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access (unless SysAdmin) + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Get all FeatureAccess records for this instance + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + from modules.datamodels.datamodelRbac import Role + from modules.datamodels.datamodelUam import UserInDB + + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"featureInstanceId": instanceId} + ) + + result = [] + for fa in featureAccesses: + userId = fa.get("userId") + featureAccessId = fa.get("id") + + # Get user info + users = rootInterface.db.getRecordset(UserInDB, recordFilter={"id": userId}) + if not users: + continue + user = users[0] + + # Get role IDs via FeatureAccessRole junction table + featureAccessRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + roleIds = [far.get("roleId") for far in featureAccessRoles] + + # Get role labels + roleLabels = [] + for roleId in roleIds: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles: + roleLabels.append(roles[0].get("roleLabel", "")) + + result.append(FeatureInstanceUserResponse( + userId=userId, + username=user.get("username", ""), + email=user.get("email"), + fullName=user.get("fullName"), + featureAccessId=featureAccessId, + roleIds=roleIds, + roleLabels=roleLabels, + enabled=fa.get("enabled", True) + )) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing feature instance users: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list feature instance users: {str(e)}" + ) + + +@router.post("/instances/{instanceId}/users", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def addUserToFeatureInstance( + request: Request, + instanceId: str, + data: FeatureInstanceUserCreate, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Add a user to a feature instance with specified roles. + + Creates a FeatureAccess record and associated FeatureAccessRole records. + + Args: + instanceId: FeatureInstance ID + data: User and role data + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to add users to feature instances" + ) + + # Verify user exists + from modules.datamodels.datamodelUam import UserInDB + users = rootInterface.db.getRecordset(UserInDB, recordFilter={"id": data.userId}) + if not users: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User '{data.userId}' not found" + ) + + # Check if user already has access + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": data.userId, "featureInstanceId": instanceId} + ) + if existingAccess: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="User already has access to this feature instance" + ) + + # Create FeatureAccess record + featureAccess = FeatureAccess( + userId=data.userId, + featureInstanceId=instanceId, + enabled=True + ) + createdAccess = rootInterface.db.recordCreate(FeatureAccess, featureAccess.model_dump()) + featureAccessId = createdAccess.get("id") + + # Create FeatureAccessRole records for each role + for roleId in data.roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + + logger.info( + f"User {context.user.id} added user {data.userId} to feature instance {instanceId} " + f"with roles {data.roleIds}" + ) + + return { + "featureAccessId": featureAccessId, + "userId": data.userId, + "featureInstanceId": instanceId, + "roleIds": data.roleIds, + "enabled": True + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding user to feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to add user to feature instance: {str(e)}" + ) + + +@router.delete("/instances/{instanceId}/users/{userId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +async def removeUserFromFeatureInstance( + request: Request, + instanceId: str, + userId: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, str]: + """ + Remove a user's access from a feature instance. + + Deletes the FeatureAccess record (CASCADE will delete FeatureAccessRole records). + + Args: + instanceId: FeatureInstance ID + userId: User ID to remove + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to remove users from feature instances" + ) + + # Find FeatureAccess record + from modules.datamodels.datamodelMembership import FeatureAccess + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) + if not existingAccess: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User does not have access to this feature instance" + ) + + featureAccessId = existingAccess[0].get("id") + + # Delete FeatureAccess (CASCADE will delete FeatureAccessRole records) + rootInterface.db.recordDelete(FeatureAccess, featureAccessId) + + logger.info( + f"User {context.user.id} removed user {userId} from feature instance {instanceId}" + ) + + return { + "message": "User access removed", + "userId": userId, + "featureInstanceId": instanceId + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error removing user from feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to remove user from feature instance: {str(e)}" + ) + + +@router.put("/instances/{instanceId}/users/{userId}/roles", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def updateFeatureInstanceUserRoles( + request: Request, + instanceId: str, + userId: str, + roleIds: List[str], + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Update a user's roles in a feature instance. + + Replaces all existing FeatureAccessRole records with new ones. + + Args: + instanceId: FeatureInstance ID + userId: User ID to update + roleIds: New list of role IDs + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to update user roles" + ) + + # Find FeatureAccess record + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) + if not existingAccess: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User does not have access to this feature instance" + ) + + featureAccessId = existingAccess[0].get("id") + + # Delete existing FeatureAccessRole records + existingRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + for role in existingRoles: + rootInterface.db.recordDelete(FeatureAccessRole, role.get("id")) + + # Create new FeatureAccessRole records + for roleId in roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + + logger.info( + f"User {context.user.id} updated roles for user {userId} in feature instance {instanceId}: {roleIds}" + ) + + return { + "featureAccessId": featureAccessId, + "userId": userId, + "featureInstanceId": instanceId, + "roleIds": roleIds + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating user roles in feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update user roles: {str(e)}" + ) + + +@router.get("/instances/{instanceId}/available-roles", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getFeatureInstanceAvailableRoles( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + Get available roles for a feature instance. + + Returns instance-specific roles (copied from templates) that can be assigned to users. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Get roles for this instance + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset( + Role, + recordFilter={"featureInstanceId": instanceId} + ) + + result = [] + for role in instanceRoles: + result.append({ + "id": role.get("id"), + "roleLabel": role.get("roleLabel"), + "description": role.get("description", {}), + "featureCode": role.get("featureCode"), + "isSystemRole": role.get("isSystemRole", False) + }) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting available roles for feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get available roles: {str(e)}" + ) + + # ============================================================================= # Dynamic Feature Route (MUST be last to avoid catching /instances, /my, etc.) # ============================================================================= diff --git a/modules/routes/routeRbac.py b/modules/routes/routeRbac.py index 03c66ae0..5592bfa1 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeRbac.py @@ -565,12 +565,20 @@ async def deleteAccessRule( async def listRoles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), + includeTemplates: bool = Query(False, description="Include feature template roles"), currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse: """ - Get list of all available roles with metadata. + Get list of global/system roles with metadata. MULTI-TENANT: SysAdmin-only (roles are system resources). + By default, only returns true global roles (mandateId=None, featureInstanceId=None, featureCode=None). + Feature template roles are managed via /api/features/templates/roles. + + Args: + pagination: Optional pagination parameters + includeTemplates: If True, also include feature template roles (featureCode != None) + Returns: - List of role dictionaries with role label, description, and user count """ @@ -598,8 +606,18 @@ async def listRoles( roleCounts = interface.countRoleAssignments() # Convert Role objects to dictionaries and add user counts + # Filter to only global roles (mandateId=None, featureInstanceId=None) + # Unless includeTemplates=True, also exclude feature template roles (featureCode != None) result = [] for role in dbRoles: + # Filter: Only global roles (no mandate, no instance) + if role.mandateId is not None or role.featureInstanceId is not None: + continue + + # Filter: Exclude feature template roles unless includeTemplates=True + if not includeTemplates and role.featureCode is not None: + continue + result.append({ "id": role.id, "roleLabel": role.roleLabel, diff --git a/tool_db_adapt_to_models.py b/scripts/script_db_adapt_to_models.py similarity index 99% rename from tool_db_adapt_to_models.py rename to scripts/script_db_adapt_to_models.py index 85c6e8fc..163c4cb8 100644 --- a/tool_db_adapt_to_models.py +++ b/scripts/script_db_adapt_to_models.py @@ -8,7 +8,7 @@ Einfaches Script das: 3. Spezialfall: UserInDB.privilege → roleLabels migriert Verwendung: - python tool_db_adapt_to_models.py [--dry-run] [--db ] + python script_db_adapt_to_models.py [--dry-run] [--db ] """ import os @@ -21,7 +21,7 @@ from typing import Dict, List, Any, Optional # Gateway-Pfad setzen scriptPath = Path(__file__).resolve() -gatewayPath = scriptPath.parent +gatewayPath = scriptPath.parent.parent sys.path.insert(0, str(gatewayPath)) os.chdir(str(gatewayPath)) diff --git a/scripts/script_db_cleanup_duplicate_roles.py b/scripts/script_db_cleanup_duplicate_roles.py new file mode 100644 index 00000000..392cde80 --- /dev/null +++ b/scripts/script_db_cleanup_duplicate_roles.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +""" +Cleanup script for duplicate roles in the database. + +This script removes duplicate Role records that were created due to the IS NULL bug +in connectorDbPostgre.py. The bug caused `mandateId = NULL` to always return FALSE, +which meant the duplicate check in bootstrap didn't work. + +Usage: + python cleanupDuplicateRoles.py + +The script will: +1. Find all duplicate roles (same roleLabel + featureCode + featureInstanceId + mandateId) +2. Keep the oldest one (first created) and delete the rest +3. Report the number of deleted roles +""" + +import sys +import os + +# Add parent directory to path +gatewayDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, gatewayDir) + +# Load environment variables from env_dev.env +from dotenv import load_dotenv +envPath = os.path.join(gatewayDir, "env_dev.env") +if os.path.exists(envPath): + load_dotenv(envPath) + +from modules.datamodels.datamodelRbac import Role +from modules.security.rootAccess import getRootDbAppConnector +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def _getDbConnector(): + """Get a database connector using the application's configuration.""" + return getRootDbAppConnector() + + +def cleanupDuplicateRoles(): + """ + Clean up duplicate roles in the database. + + Keeps the first role (by ID, which is UUID-based) for each unique combination of: + - roleLabel + - featureCode + - featureInstanceId + - mandateId + """ + db = _getDbConnector() + + # Get all roles + allRoles = db.getRecordset(Role, recordFilter=None) + logger.info(f"Found {len(allRoles)} total roles in database") + + # Group roles by their unique key + roleGroups = {} + for role in allRoles: + # Create a key tuple for grouping + # Note: None values need special handling for dict keys + key = ( + role.get("roleLabel"), + role.get("featureCode") or "__NONE__", + role.get("featureInstanceId") or "__NONE__", + role.get("mandateId") or "__NONE__" + ) + + if key not in roleGroups: + roleGroups[key] = [] + roleGroups[key].append(role) + + # Find and delete duplicates + deletedCount = 0 + for key, roles in roleGroups.items(): + if len(roles) > 1: + # Sort by ID (UUID, string comparison works for finding "first") + # Actually, we want to keep one - let's keep by created order if available + # Since there's no createdAt, we'll just keep the first one + toKeep = roles[0] + toDelete = roles[1:] + + logger.info(f"Found {len(roles)} duplicates for key {key}") + logger.info(f" Keeping: {toKeep.get('id')} ({toKeep.get('roleLabel')})") + + for role in toDelete: + roleId = role.get("id") + try: + db.recordDelete(Role, roleId) + deletedCount += 1 + logger.info(f" Deleted: {roleId}") + except Exception as e: + logger.error(f" Failed to delete {roleId}: {e}") + + logger.info(f"Cleanup complete: {deletedCount} duplicate roles deleted") + logger.info(f"Remaining roles: {len(allRoles) - deletedCount}") + + return deletedCount + + +def showRoleSummary(): + """Show a summary of roles grouped by type.""" + db = _getDbConnector() + + allRoles = db.getRecordset(Role, recordFilter=None) + + # Categorize roles + systemRoles = [] + templateRoles = [] + mandateRoles = [] + instanceRoles = [] + + for role in allRoles: + mandateId = role.get("mandateId") + featureInstanceId = role.get("featureInstanceId") + featureCode = role.get("featureCode") + isSystemRole = role.get("isSystemRole", False) + + if isSystemRole: + systemRoles.append(role) + elif mandateId is None and featureInstanceId is None and featureCode: + templateRoles.append(role) + elif mandateId is None and featureInstanceId is None and not featureCode: + # Global non-system role (shouldn't exist normally) + systemRoles.append(role) + elif mandateId and featureInstanceId is None: + mandateRoles.append(role) + elif featureInstanceId: + instanceRoles.append(role) + + print("\n" + "=" * 60) + print("ROLE SUMMARY") + print("=" * 60) + + print(f"\n1. SYSTEM ROLES ({len(systemRoles)}):") + for r in systemRoles: + print(f" - {r.get('roleLabel')} (isSystemRole={r.get('isSystemRole')})") + + print(f"\n2. TEMPLATE ROLES ({len(templateRoles)}) - (mandateId=NULL, featureInstanceId=NULL, featureCode!=NULL):") + templateByFeature = {} + for r in templateRoles: + fc = r.get("featureCode") + if fc not in templateByFeature: + templateByFeature[fc] = [] + templateByFeature[fc].append(r) + + for fc, roles in sorted(templateByFeature.items()): + print(f" [{fc}] ({len(roles)} roles):") + for r in roles: + print(f" - {r.get('roleLabel')}") + + print(f"\n3. MANDATE ROLES ({len(mandateRoles)}) - (mandateId!=NULL, featureInstanceId=NULL):") + for r in mandateRoles[:10]: # Show max 10 + print(f" - {r.get('roleLabel')} (mandate: {r.get('mandateId')[:8]}...)") + if len(mandateRoles) > 10: + print(f" ... and {len(mandateRoles) - 10} more") + + print(f"\n4. INSTANCE ROLES ({len(instanceRoles)}) - (featureInstanceId!=NULL):") + for r in instanceRoles[:10]: # Show max 10 + print(f" - {r.get('roleLabel')} (instance: {r.get('featureInstanceId')[:8]}...)") + if len(instanceRoles) > 10: + print(f" ... and {len(instanceRoles) - 10} more") + + print("\n" + "=" * 60) + print(f"TOTAL: {len(allRoles)} roles") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Cleanup duplicate roles in database") + parser.add_argument("--summary", action="store_true", help="Show role summary without deleting") + parser.add_argument("--cleanup", action="store_true", help="Delete duplicate roles") + + args = parser.parse_args() + + if args.summary: + showRoleSummary() + elif args.cleanup: + cleanupDuplicateRoles() + showRoleSummary() + else: + # Default: show summary only + showRoleSummary() + print("\nTo delete duplicates, run with --cleanup flag") diff --git a/tool_db_export_migration.py b/scripts/script_db_export_migration.py similarity index 97% rename from tool_db_export_migration.py rename to scripts/script_db_export_migration.py index aea697c1..dd3cc940 100644 --- a/tool_db_export_migration.py +++ b/scripts/script_db_export_migration.py @@ -16,7 +16,7 @@ Datenbanken: - poweron_trustee (Trustee Daten) Verwendung: - python tool_db_export_migration.py [--output ] [--pretty] + python script_db_export_migration.py [--output ] [--pretty] Optionen: --output, -o Pfad zur Ausgabedatei (Standard: migration_export_.json) @@ -40,23 +40,26 @@ from pathlib import Path # Find gateway directory (could be in local/pending/ or gateway/) scriptPath = Path(__file__).resolve() gatewayPath = scriptPath.parent +# If we're in scripts/, go up to gateway/ +if gatewayPath.name == "scripts": + gatewayPath = gatewayPath.parent # If we're in local/pending/, go up to find gateway/ -if gatewayPath.name == "pending": +elif gatewayPath.name == "pending": gatewayPath = gatewayPath.parent.parent / "gateway" elif gatewayPath.name == "local": gatewayPath = gatewayPath.parent / "gateway" # If gateway doesn't exist, try current directory if not gatewayPath.exists(): - gatewayPath = Path(__file__).parent.parent.parent / "gateway" + gatewayPath = scriptPath.parent.parent.parent / "gateway" if gatewayPath.exists(): sys.path.insert(0, str(gatewayPath)) # Change working directory to gateway so APP_CONFIG can find .env file os.chdir(str(gatewayPath)) else: # Fallback: assume we're already in gateway/ or add parent - sys.path.insert(0, str(Path(__file__).parent)) + sys.path.insert(0, str(scriptPath.parent)) # Try to change to gateway directory if it exists - potentialGateway = Path(__file__).parent + potentialGateway = scriptPath.parent if potentialGateway.exists() and (potentialGateway / "modules" / "shared" / "configuration.py").exists(): os.chdir(str(potentialGateway)) @@ -579,7 +582,7 @@ def exportDatabase( if logDir and os.path.isabs(logDir): outputDir = logDir else: - outputDir = os.path.join(os.path.dirname(__file__), "local", "logs") + outputDir = os.path.join(str(gatewayPath), "local", "logs") os.makedirs(outputDir, exist_ok=True) outputPath = os.path.join(outputDir, f"migration_export_{timestamp}.json") @@ -751,12 +754,12 @@ Datenbanken: poweron_trustee - Trustee Daten Beispiele: - python tool_db_export_migration.py - python tool_db_export_migration.py --pretty - python tool_db_export_migration.py -o backup.json --pretty - python tool_db_export_migration.py --db poweron_app,poweron_chat - python tool_db_export_migration.py --exclude Token,AuthEvent --include-meta - python tool_db_export_migration.py --summary + python script_db_export_migration.py + python script_db_export_migration.py --pretty + python script_db_export_migration.py -o backup.json --pretty + python script_db_export_migration.py --db poweron_app,poweron_chat + python script_db_export_migration.py --exclude Token,AuthEvent --include-meta + python script_db_export_migration.py --summary """ ) diff --git a/tool_security_encrypt_all_env_files.py b/scripts/script_security_encrypt_all_env_files.py similarity index 95% rename from tool_security_encrypt_all_env_files.py rename to scripts/script_security_encrypt_all_env_files.py index 420591dd..9981ad52 100644 --- a/tool_security_encrypt_all_env_files.py +++ b/scripts/script_security_encrypt_all_env_files.py @@ -10,16 +10,16 @@ keys for each environment. Usage: # Encrypt all secrets in all environment files - python tool_security_encrypt_all_env_files.py + python script_security_encrypt_all_env_files.py # Dry run - show what would be changed without making changes - python tool_security_encrypt_all_env_files.py --dry-run + python script_security_encrypt_all_env_files.py --dry-run # Skip backup creation - python tool_security_encrypt_all_env_files.py --no-backup + python script_security_encrypt_all_env_files.py --no-backup # Process only specific environment files - python tool_security_encrypt_all_env_files.py --files env_dev.env env_prod.env + python script_security_encrypt_all_env_files.py --files env_dev.env env_prod.env """ import sys @@ -29,14 +29,15 @@ from pathlib import Path from datetime import datetime from typing import List, Dict, Any -# Add the modules directory to the Python path -current_dir = Path(__file__).parent -modules_dir = current_dir / 'modules' -if modules_dir.exists(): - sys.path.insert(0, str(modules_dir)) +# Add the gateway directory to the Python path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +modulesDir = gatewayPath / "modules" +if modulesDir.exists(): + sys.path.insert(0, str(gatewayPath)) else: - print(f"Error: Modules directory not found: {modules_dir}") - print(f"Make sure you're running this script from the gateway directory") + print(f"Error: Modules directory not found: {modulesDir}") + print("Make sure you're running this script from the gateway directory") sys.exit(1) # Import encryption functions @@ -45,7 +46,7 @@ try: except ImportError as e: print(f"Error: Could not import encryption functions from shared.configuration: {e}") print(f"Make sure you're running this script from the gateway directory") - print(f"Modules directory: {modules_dir}") + print(f"Modules directory: {modulesDir}") sys.exit(1) def get_env_type_from_file(file_path: Path) -> str: diff --git a/tool_security_encrypt_config_value.py b/scripts/script_security_encrypt_config_value.py similarity index 96% rename from tool_security_encrypt_config_value.py rename to scripts/script_security_encrypt_config_value.py index e082e541..afaeb827 100644 --- a/tool_security_encrypt_config_value.py +++ b/scripts/script_security_encrypt_config_value.py @@ -10,18 +10,18 @@ It can also encrypt all *_SECRET keys in an environment file at once. Usage: # Encrypt a single value - python tool_encrypt_config_value.py --value "my_secret_value" --env dev - python tool_encrypt_config_value.py --file "path/to/file.json" --env prod + python script_security_encrypt_config_value.py --value "my_secret_value" --env dev + python script_security_encrypt_config_value.py --file "path/to/file.json" --env prod # Encrypt all secrets in a file - python tool_encrypt_config_value.py --encrypt-all env_dev.env --env dev - python tool_encrypt_config_value.py --encrypt-all env_prod.env --env prod --dry-run + python script_security_encrypt_config_value.py --encrypt-all env_dev.env --env dev + python script_security_encrypt_config_value.py --encrypt-all env_prod.env --env prod --dry-run # Decrypt a value (for testing) - python tool_encrypt_config_value.py --decrypt "DEV_ENC:encrypted_value" + python script_security_encrypt_config_value.py --decrypt "DEV_ENC:encrypted_value" # Verify master key is correct - python tool_encrypt_config_value.py --verify "PROD_ENC:Z0FBQUFBQm8xSU5p..." + python script_security_encrypt_config_value.py --verify "PROD_ENC:Z0FBQUFBQm8xSU5p..." """ import sys @@ -32,8 +32,11 @@ import shutil from pathlib import Path from datetime import datetime -# Add the modules directory to the Python path -sys.path.insert(0, str(Path(__file__).parent / 'modules')) +# Add the gateway directory to the Python path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +projectRoot = gatewayPath.parent +sys.path.insert(0, str(gatewayPath)) from modules.shared.configuration import encryptValue, decryptValue, _isEncryptedValue as isEncryptedValue @@ -412,7 +415,7 @@ def main(): key_source = f"file '{key_location}'" else: # Try default key file location - default_key_file = Path(__file__).parent.parent / 'local' / 'key.txt' + default_key_file = projectRoot / "local" / "key.txt" if default_key_file.exists(): print(f" [OK] Found master key in default file: {default_key_file}") key_source = f"file '{default_key_file}'" diff --git a/tool_security_generate_master_keys.py b/scripts/script_security_generate_master_keys.py similarity index 88% rename from tool_security_generate_master_keys.py rename to scripts/script_security_generate_master_keys.py index a5426e4f..6da55d26 100644 --- a/tool_security_generate_master_keys.py +++ b/scripts/script_security_generate_master_keys.py @@ -8,8 +8,8 @@ This tool generates cryptographically secure 256-bit master keys for all environ and updates the key.txt file with the new keys. Usage: - python generate_master_keys.py - python generate_master_keys.py --output "path/to/key.txt" + python script_security_generate_master_keys.py + python script_security_generate_master_keys.py --output "path/to/key.txt" """ import sys @@ -19,6 +19,10 @@ import base64 import argparse from pathlib import Path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +projectRoot = gatewayPath.parent + def generate_master_key(): """Generate a secure 256-bit master key.""" # Generate 32 random bytes (256 bits) @@ -29,8 +33,8 @@ def generate_master_key(): def main(): parser = argparse.ArgumentParser(description='Generate secure master keys for all environments') parser.add_argument('--output', '-o', - default='../local/key.txt', - help='Output file path (default: ../local/key.txt)') + default=str(projectRoot / "local" / "key.txt"), + help='Output file path (default: poweron/local/key.txt)') parser.add_argument('--force', '-f', action='store_true', help='Overwrite existing key file without confirmation') diff --git a/tool_stats_durations_from_log.py b/scripts/script_stats_durations_from_log.py similarity index 100% rename from tool_stats_durations_from_log.py rename to scripts/script_stats_durations_from_log.py diff --git a/tool_stats_get_codelines.py b/scripts/script_stats_get_codelines.py similarity index 100% rename from tool_stats_get_codelines.py rename to scripts/script_stats_get_codelines.py diff --git a/tool_stats_showUnusedFunctions.py b/scripts/script_stats_showUnusedFunctions.py similarity index 96% rename from tool_stats_showUnusedFunctions.py rename to scripts/script_stats_showUnusedFunctions.py index 22dd5144..f7f6b2c3 100644 --- a/tool_stats_showUnusedFunctions.py +++ b/scripts/script_stats_showUnusedFunctions.py @@ -191,18 +191,19 @@ class FunctionUsageAnalyzer: def main(): """Main function to run the analysis.""" - # Get the directory where this script is located - script_dir = Path(__file__).parent - logger.info(f"Analyzing codebase in: {script_dir}") + # Get the gateway directory for a full codebase scan + scriptPath = Path(__file__).resolve() + gatewayPath = scriptPath.parent.parent + logger.info(f"Analyzing codebase in: {gatewayPath}") - analyzer = FunctionUsageAnalyzer(script_dir) + analyzer = FunctionUsageAnalyzer(gatewayPath) analyzer.analyze_codebase() report = analyzer.generate_report() print(report) # Save report to file - report_file = script_dir / "unused_functions_report.txt" + report_file = gatewayPath / "unused_functions_report.txt" with open(report_file, 'w', encoding='utf-8') as f: f.write(report) From b0d89b90804323614d022f914252803f53ba2292 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 10:59:44 +0100 Subject: [PATCH 07/32] fixed uid mapping to id --- modules/routes/routeDataMandates.py | 10 +++++----- modules/routes/routeFeatures.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index ec109c6b..90948b8a 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -43,7 +43,7 @@ class UserMandateCreate(BaseModel): class UserMandateResponse(BaseModel): """Response model for user mandate membership""" - userMandateId: str + id: str # UserMandate ID as primary key userId: str mandateId: str roleIds: List[str] @@ -52,11 +52,11 @@ class UserMandateResponse(BaseModel): class MandateUserInfo(BaseModel): """User info within a mandate context""" + id: str # UserMandate ID as primary key userId: str username: str email: Optional[str] fullName: Optional[str] - userMandateId: str roleIds: List[str] roleLabels: List[str] # Resolved role labels for display enabled: bool @@ -379,11 +379,11 @@ async def listMandateUsers( roleLabels.append(roleId) # Fallback to ID if not found result.append(MandateUserInfo( + id=um.get("id"), # UserMandate ID as primary key userId=str(user.id), username=user.username, email=user.email, fullName=user.fullName, - userMandateId=um.get("id"), roleIds=roleIds, roleLabels=roleLabels, enabled=um.get("enabled", True) @@ -500,7 +500,7 @@ async def addUserToMandate( ) return UserMandateResponse( - userMandateId=str(userMandate.id), + id=str(userMandate.id), # UserMandate ID as primary key userId=data.targetUserId, mandateId=targetMandateId, roleIds=data.roleIds, @@ -690,7 +690,7 @@ async def updateUserRolesInMandate( ) return UserMandateResponse( - userMandateId=str(membership.id), + id=str(membership.id), # UserMandate ID as primary key userId=targetUserId, mandateId=targetMandateId, roleIds=roleIds, diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index a9a124a8..43d120f1 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -722,11 +722,11 @@ class FeatureInstanceUserCreate(BaseModel): class FeatureInstanceUserResponse(BaseModel): """Response model for a user in a feature instance""" + id: str # Use the FeatureAccess ID as primary key userId: str username: str email: Optional[str] fullName: Optional[str] - featureAccessId: str roleIds: List[str] roleLabels: List[str] enabled: bool @@ -803,11 +803,11 @@ async def listFeatureInstanceUsers( roleLabels.append(roles[0].get("roleLabel", "")) result.append(FeatureInstanceUserResponse( + id=featureAccessId, # FeatureAccess ID as primary key userId=userId, username=user.get("username", ""), email=user.get("email"), fullName=user.get("fullName"), - featureAccessId=featureAccessId, roleIds=roleIds, roleLabels=roleLabels, enabled=fa.get("enabled", True) From 6c8c703115ec4fdeabe969cbeacfb690e7c9bfb3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 11:26:19 +0100 Subject: [PATCH 08/32] proper junction table handling --- app.py | 31 +++++---- modules/routes/routeFeatures.py | 117 ++++++++++++++++++++------------ 2 files changed, 90 insertions(+), 58 deletions(-) diff --git a/app.py b/app.py index b489194d..472de2a1 100644 --- a/app.py +++ b/app.py @@ -145,21 +145,24 @@ def initLogging(): def filter(self, record): if isinstance(record.msg, str): # Remove only emojis, preserve other Unicode characters like quotes - - # Remove emoji characters specifically - record.msg = "".join( - char - for char in record.msg - if unicodedata.category(char) != "So" - or not ( - 0x1F600 <= ord(char) <= 0x1F64F - or 0x1F300 <= ord(char) <= 0x1F5FF - or 0x1F680 <= ord(char) <= 0x1F6FF - or 0x1F1E0 <= ord(char) <= 0x1F1FF - or 0x2600 <= ord(char) <= 0x26FF - or 0x2700 <= ord(char) <= 0x27BF + # Guard against None characters during shutdown + try: + record.msg = "".join( + char + for char in record.msg + if char is not None and unicodedata.category(char) != "So" + or (char is not None and not ( + 0x1F600 <= ord(char) <= 0x1F64F + or 0x1F300 <= ord(char) <= 0x1F5FF + or 0x1F680 <= ord(char) <= 0x1F6FF + or 0x1F1E0 <= ord(char) <= 0x1F1FF + or 0x2600 <= ord(char) <= 0x26FF + or 0x2700 <= ord(char) <= 0x27BF + )) ) - ) + except (TypeError, AttributeError): + # Handle edge cases during shutdown + pass return True # Add filter to normalize problematic unicode (e.g., arrows) to ASCII for terminals like cp1252 diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 43d120f1..0659d919 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -199,19 +199,28 @@ async def getMyFeatureInstances( def _getUserRoleInInstance(rootInterface, userId: str, instanceId: str) -> str: """Get the user's primary role label in a feature instance.""" try: - from modules.datamodels.datamodelRbac import UserRole, Role + from modules.datamodels.datamodelRbac import Role + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole - # Get user-role assignments for this instance - userRoles = rootInterface.db.getRecordset( - UserRole, - recordFilter={"userId": userId} + # Get FeatureAccess for this user and instance + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} ) - for ur in userRoles: - roleId = ur.get("roleId") - if roleId: + if featureAccesses: + featureAccessId = featureAccesses[0].get("id") + + # Get role IDs via FeatureAccessRole junction table + featureAccessRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + + if featureAccessRoles: + roleId = featureAccessRoles[0].get("roleId") roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) - if roles and str(roles[0].get("featureInstanceId")) == instanceId: + if roles: return roles[0].get("roleLabel", "user") return "user" # Default @@ -230,52 +239,72 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict } try: - from modules.datamodels.datamodelRbac import UserRole, Role, RolePermission + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole - # Get user's roles for this instance - userRoles = rootInterface.db.getRecordset(UserRole, recordFilter={"userId": userId}) - roleIds = [] + # Get FeatureAccess for this user and instance + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) - for ur in userRoles: - roleId = ur.get("roleId") - if roleId: - roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) - if roles and str(roles[0].get("featureInstanceId")) == instanceId: - roleIds.append(roleId) + if not featureAccesses: + return permissions + + # Get role IDs via FeatureAccessRole junction table + featureAccessId = featureAccesses[0].get("id") + featureAccessRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + roleIds = [far.get("roleId") for far in featureAccessRoles] if not roleIds: return permissions - # Get permissions for all roles + # Get permissions (AccessRules) for all roles for roleId in roleIds: - rolePerms = rootInterface.db.getRecordset( - RolePermission, + accessRules = rootInterface.db.getRecordset( + AccessRule, recordFilter={"roleId": roleId} ) - for perm in rolePerms: - tableName = perm.get("tableName", "") - if tableName: - if tableName not in permissions["tables"]: - permissions["tables"][tableName] = { - "view": False, - "read": "n", - "create": "n", - "update": "n", - "delete": "n" - } - - # Merge permissions (highest wins) - current = permissions["tables"][tableName] - current["view"] = current["view"] or perm.get("canView", False) - current["read"] = _mergeAccessLevel(current["read"], perm.get("readLevel", "n")) - current["create"] = _mergeAccessLevel(current["create"], perm.get("createLevel", "n")) - current["update"] = _mergeAccessLevel(current["update"], perm.get("updateLevel", "n")) - current["delete"] = _mergeAccessLevel(current["delete"], perm.get("deleteLevel", "n")) + for rule in accessRules: + context = rule.get("context", "") + item = rule.get("item", "") - viewName = perm.get("viewName", "") - if viewName: - permissions["views"][viewName] = permissions["views"].get(viewName, False) or perm.get("canAccess", False) + # Handle DATA context (tables/fields) + if context == "DATA" or context == AccessRuleContext.DATA: + if item: + # Check if it's a field (table.field) or table + if "." in item: + tableName, fieldName = item.split(".", 1) + if fieldName not in permissions["fields"]: + permissions["fields"][fieldName] = {"view": False} + permissions["fields"][fieldName]["view"] = permissions["fields"][fieldName]["view"] or rule.get("view", False) + else: + tableName = item + if tableName not in permissions["tables"]: + permissions["tables"][tableName] = { + "view": False, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + } + + # Merge permissions (highest wins) + current = permissions["tables"][tableName] + current["view"] = current["view"] or rule.get("view", False) + current["read"] = _mergeAccessLevel(current["read"], rule.get("read") or "n") + current["create"] = _mergeAccessLevel(current["create"], rule.get("create") or "n") + current["update"] = _mergeAccessLevel(current["update"], rule.get("update") or "n") + current["delete"] = _mergeAccessLevel(current["delete"], rule.get("delete") or "n") + + # Handle UI context (views) + elif context == "UI" or context == AccessRuleContext.UI: + if item: + permissions["views"][item] = permissions["views"].get(item, False) or rule.get("view", False) return permissions From ac88f25526338d88699a5fa3ad9758332bf216c6 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 15:57:16 +0100 Subject: [PATCH 09/32] serverside filter and sort for form generic --- modules/interfaces/interfaceBootstrap.py | 187 +++++++++++++++++++++++ modules/routes/routeDataMandates.py | 110 +++++++++++-- modules/routes/routeRbac.py | 68 +++++++-- 3 files changed, 343 insertions(+), 22 deletions(-) diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 545b0040..cbf8295a 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -62,6 +62,9 @@ def initBootstrap(db: DatabaseConnector) -> None: # Initialize RBAC rules (uses roleIds from roles) initRbacRules(db) + # Initialize AccessRules for feature-template roles (idempotent - adds missing rules) + _initFeatureTemplateRoleAccessRules(db) + # Initialize admin user adminUserId = initAdminUser(db, mandateId) @@ -498,6 +501,190 @@ def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: return None +def _initFeatureTemplateRoleAccessRules(db: DatabaseConnector) -> None: + """ + Initialize AccessRules for feature-template roles. + This is idempotent - only adds rules that don't exist yet. + + Feature-template roles need explicit AccessRules for their respective tables: + - trustee-admin/accountant/client -> TrusteeOrganisation, TrusteeContract, etc. + - chatbot-admin/user -> ChatSession, etc. + - workflow-admin/editor/viewer -> ChatWorkflow, etc. + + Args: + db: Database connector instance + """ + logger.info("Checking feature-template role AccessRules") + + # Define feature-specific table access + featureTableAccess = { + "trustee": { + "tables": [ + "TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", + "TrusteeContract", "TrusteeDocument", "TrusteePosition", + "TrusteePositionDocument" + ], + "roles": { + "trustee-admin": { + "view": True, + "read": AccessLevel.ALL, + "create": AccessLevel.ALL, + "update": AccessLevel.ALL, + "delete": AccessLevel.ALL + }, + "trustee-accountant": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.GROUP, + "update": AccessLevel.GROUP, + "delete": AccessLevel.NONE + }, + "trustee-client": { + "view": True, + "read": AccessLevel.MY, + "create": AccessLevel.NONE, + "update": AccessLevel.NONE, + "delete": AccessLevel.NONE + } + } + }, + "chatbot": { + "tables": ["ChatSession", "ChatMessage"], + "roles": { + "chatbot-admin": { + "view": True, + "read": AccessLevel.ALL, + "create": AccessLevel.ALL, + "update": AccessLevel.ALL, + "delete": AccessLevel.ALL + }, + "chatbot-user": { + "view": True, + "read": AccessLevel.MY, + "create": AccessLevel.MY, + "update": AccessLevel.MY, + "delete": AccessLevel.MY + } + } + }, + "chatworkflow": { + "tables": ["ChatWorkflow", "AutomationDefinition"], + "roles": { + "workflow-admin": { + "view": True, + "read": AccessLevel.ALL, + "create": AccessLevel.ALL, + "update": AccessLevel.ALL, + "delete": AccessLevel.ALL + }, + "workflow-editor": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.GROUP, + "update": AccessLevel.GROUP, + "delete": AccessLevel.NONE + }, + "workflow-viewer": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.NONE, + "update": AccessLevel.NONE, + "delete": AccessLevel.NONE + } + } + }, + "neutralization": { + "tables": ["DataNeutraliserConfig", "DataNeutralizerAttributes"], + "roles": { + "neutralization-admin": { + "view": True, + "read": AccessLevel.ALL, + "create": AccessLevel.ALL, + "update": AccessLevel.ALL, + "delete": AccessLevel.ALL + }, + "neutralization-analyst": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.NONE, + "update": AccessLevel.NONE, + "delete": AccessLevel.NONE + } + } + }, + "realestate": { + "tables": ["Projekt", "Parzelle", "Dokument", "Gemeinde", "Kanton", "Land"], + "roles": { + "realestate-admin": { + "view": True, + "read": AccessLevel.ALL, + "create": AccessLevel.ALL, + "update": AccessLevel.ALL, + "delete": AccessLevel.ALL + }, + "realestate-manager": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.GROUP, + "update": AccessLevel.GROUP, + "delete": AccessLevel.NONE + }, + "realestate-viewer": { + "view": True, + "read": AccessLevel.GROUP, + "create": AccessLevel.NONE, + "update": AccessLevel.NONE, + "delete": AccessLevel.NONE + } + } + } + } + + createdCount = 0 + + for featureCode, featureConfig in featureTableAccess.items(): + tables = featureConfig["tables"] + roles = featureConfig["roles"] + + for roleLabel, permissions in roles.items(): + roleId = _getRoleId(db, roleLabel) + if not roleId: + continue + + for tableName in tables: + # Check if rule already exists + existingRules = db.getRecordset( + AccessRule, + recordFilter={ + "roleId": roleId, + "context": AccessRuleContext.DATA, + "item": tableName + } + ) + + if existingRules: + continue # Rule already exists + + # Create new rule + rule = AccessRule( + roleId=roleId, + context=AccessRuleContext.DATA, + item=tableName, + view=permissions["view"], + read=permissions["read"], + create=permissions["create"], + update=permissions["update"], + delete=permissions["delete"] + ) + db.recordCreate(AccessRule, rule) + createdCount += 1 + + if createdCount > 0: + logger.info(f"Created {createdCount} feature-template role AccessRules") + else: + logger.debug("All feature-template role AccessRules already exist") + + def initRbacRules(db: DatabaseConnector) -> None: """ Initialize RBAC rules if they don't exist. diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 90948b8a..82ed3ad6 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -323,17 +323,21 @@ async def delete_mandate( # User Management within Mandates (Mandate-Admin) # ============================================================================= -@router.get("/{targetMandateId}/users", response_model=List[MandateUserInfo]) +@router.get("/{targetMandateId}/users") @limiter.limit("60/minute") async def listMandateUsers( request: Request, targetMandateId: str = Path(..., description="ID of the mandate"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext) -) -> List[MandateUserInfo]: +): """ - List all users in a mandate. + List all users in a mandate with pagination, search, and sorting support. Requires Mandate-Admin role or SysAdmin. + + Args: + pagination: Optional pagination parameters (page, pageSize, search, filters, sort) """ # Check permission if not _hasMandateAdminRole(context, targetMandateId) and not context.isSysAdmin: @@ -353,6 +357,26 @@ async def listMandateUsers( detail=f"Mandate {targetMandateId} not found" ) + # Parse pagination parameter + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + # Normalize pagination dict + if 'sort' in paginationDict and paginationDict['sort']: + normalizedSort = [] + for item in paginationDict['sort']: + if isinstance(item, dict): + normalizedSort.append(item) + paginationDict['sort'] = normalizedSort if normalizedSort else None + paginationParams = paginationDict + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException( + status_code=400, + detail=f"Invalid pagination parameter: {str(e)}" + ) + # Get all UserMandate entries for this mandate userMandates = rootInterface.db.getRecordset( UserMandate, @@ -378,17 +402,77 @@ async def listMandateUsers( else: roleLabels.append(roleId) # Fallback to ID if not found - result.append(MandateUserInfo( - id=um.get("id"), # UserMandate ID as primary key - userId=str(user.id), - username=user.username, - email=user.email, - fullName=user.fullName, - roleIds=roleIds, - roleLabels=roleLabels, - enabled=um.get("enabled", True) - )) + result.append({ + "id": um.get("id"), # UserMandate ID as primary key + "userId": str(user.id), + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "roleIds": roleIds, + "roleLabels": roleLabels, + "enabled": um.get("enabled", True) + }) + # Apply search, filtering, and sorting if pagination requested + if paginationParams: + # Apply search (if search term provided) + searchTerm = paginationParams.get('search', '').lower() if paginationParams.get('search') else '' + if searchTerm: + searchedResult = [] + for item in result: + username = (item.get("username") or "").lower() + email = (item.get("email") or "").lower() + fullName = (item.get("fullName") or "").lower() + roleLabelsStr = " ".join(item.get("roleLabels") or []).lower() + + if searchTerm in username or searchTerm in email or searchTerm in fullName or searchTerm in roleLabelsStr: + searchedResult.append(item) + result = searchedResult + + # Apply filters (if filters provided) + filters = paginationParams.get('filters') + if filters: + for fieldName, filterValue in filters.items(): + if filterValue is not None and filterValue != '': + filterValueLower = str(filterValue).lower() + result = [ + item for item in result + if str(item.get(fieldName, '')).lower() == filterValueLower + ] + + # Apply sorting + sortFields = paginationParams.get('sort') + if sortFields: + for sortItem in reversed(sortFields): + field = sortItem.get('field') + direction = sortItem.get('direction', 'asc') + if field: + result = sorted( + result, + key=lambda x: str(x.get(field, '') or '').lower(), + reverse=(direction == 'desc') + ) + + # Apply pagination + page = paginationParams.get('page', 1) + pageSize = paginationParams.get('pageSize', 25) + totalItems = len(result) + totalPages = (totalItems + pageSize - 1) // pageSize if totalItems > 0 else 0 + startIdx = (page - 1) * pageSize + endIdx = startIdx + pageSize + paginatedResult = result[startIdx:endIdx] + + return { + "items": paginatedResult, + "pagination": { + "currentPage": page, + "pageSize": pageSize, + "totalItems": totalItems, + "totalPages": totalPages + } + } + + # No pagination - return all users as list return result except HTTPException: diff --git a/modules/routes/routeRbac.py b/modules/routes/routeRbac.py index 5592bfa1..7ab9b229 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeRbac.py @@ -566,21 +566,25 @@ async def listRoles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), includeTemplates: bool = Query(False, description="Include feature template roles"), + mandateId: Optional[str] = Query(None, description="Include mandate-specific roles for this mandate"), + scopeFilter: Optional[str] = Query(None, description="Filter by scope: 'all', 'mandate', 'global', 'system'"), currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse: """ - Get list of global/system roles with metadata. + Get list of roles with metadata. MULTI-TENANT: SysAdmin-only (roles are system resources). By default, only returns true global roles (mandateId=None, featureInstanceId=None, featureCode=None). Feature template roles are managed via /api/features/templates/roles. Args: - pagination: Optional pagination parameters + pagination: Optional pagination parameters (includes search, filters, sort) includeTemplates: If True, also include feature template roles (featureCode != None) + mandateId: If provided, also include mandate-specific roles for this mandate + scopeFilter: Filter by scope type: 'all', 'mandate', 'global', 'system' Returns: - - List of role dictionaries with role label, description, and user count + - List of role dictionaries with role label, description, user count, and computed scopeType """ try: interface = getRootInterface() @@ -605,19 +609,45 @@ async def listRoles( # Count role assignments from UserMandateRole table roleCounts = interface.countRoleAssignments() + # Helper function to compute scopeType + def _computeScopeType(role) -> str: + if role.isSystemRole: + return "system" + if role.mandateId: + return "mandate" + return "global" + # Convert Role objects to dictionaries and add user counts - # Filter to only global roles (mandateId=None, featureInstanceId=None) - # Unless includeTemplates=True, also exclude feature template roles (featureCode != None) + # Filter logic: + # - Always include global roles (mandateId=None, featureInstanceId=None) + # - If mandateId provided, also include roles for that specific mandate + # - Unless includeTemplates=True, exclude feature template roles (featureCode != None) result = [] for role in dbRoles: - # Filter: Only global roles (no mandate, no instance) - if role.mandateId is not None or role.featureInstanceId is not None: + # Always exclude feature-instance level roles + if role.featureInstanceId is not None: + continue + + # Include global roles (mandateId=None) OR mandate-specific roles if mandateId matches + if role.mandateId is not None and (mandateId is None or role.mandateId != mandateId): continue # Filter: Exclude feature template roles unless includeTemplates=True if not includeTemplates and role.featureCode is not None: continue + # Compute scopeType (system, global, mandate) + scopeType = _computeScopeType(role) + + # Apply scopeFilter if provided + if scopeFilter and scopeFilter != 'all': + if scopeFilter == 'mandate' and scopeType != 'mandate': + continue + if scopeFilter == 'global' and scopeType not in ('global', 'system'): + continue + if scopeFilter == 'system' and scopeType != 'system': + continue + result.append({ "id": role.id, "roleLabel": role.roleLabel, @@ -626,11 +656,31 @@ async def listRoles( "featureInstanceId": role.featureInstanceId, "featureCode": role.featureCode, "userCount": roleCounts.get(str(role.id), 0), - "isSystemRole": role.isSystemRole + "isSystemRole": role.isSystemRole, + "scopeType": scopeType # Computed field for frontend display }) - # Apply filtering and sorting if pagination requested + # Apply search, filtering and sorting if pagination requested if paginationParams: + # Apply search (if search term provided in filters) + searchTerm = paginationParams.filters.get("search", "").lower() if paginationParams.filters else "" + if searchTerm: + searchedResult = [] + for item in result: + # Search in roleLabel and description + roleLabel = (item.get("roleLabel") or "").lower() + description = item.get("description") + descText = "" + if isinstance(description, dict): + descText = " ".join(str(v) for v in description.values()).lower() + elif description: + descText = str(description).lower() + scopeType = (item.get("scopeType") or "").lower() + + if searchTerm in roleLabel or searchTerm in descText or searchTerm in scopeType: + searchedResult.append(item) + result = searchedResult + # Apply filtering (if filters provided) if paginationParams.filters: # Use the interface's filter method From 5fcbc6acd3d70503a72f89385d473e23f6aa524a Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 21:19:52 +0100 Subject: [PATCH 10/32] saas multi mandate tested --- modules/routes/routeDataUsers.py | 126 ++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 4 deletions(-) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 81115e51..62a9b344 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -27,6 +27,118 @@ from modules.datamodels.datamodelPagination import PaginationParams, PaginatedRe # Configure logger logger = logging.getLogger(__name__) + +def _applyFiltersAndSort(items: List[Dict[str, Any]], paginationParams: Optional[PaginationParams]) -> List[Dict[str, Any]]: + """ + Apply filters and sorting to a list of items. + This is used when we can't do server-side filtering in the database (e.g., SysAdmin view). + + Args: + items: List of dictionaries to filter/sort + paginationParams: Pagination parameters with filters and sort + + Returns: + Filtered and sorted list + """ + if not paginationParams: + return items + + result = items.copy() + + # Apply filters + if paginationParams.filters: + filters = paginationParams.filters + + # Handle general search + searchTerm = filters.get('search', '').lower() if filters.get('search') else None + + if searchTerm: + def matchesSearch(item: Dict[str, Any]) -> bool: + for value in item.values(): + if value is not None and searchTerm in str(value).lower(): + return True + return False + result = [item for item in result if matchesSearch(item)] + + # Handle field-specific filters + for field, filterValue in filters.items(): + if field == 'search': + continue # Already handled + + if isinstance(filterValue, dict) and 'operator' in filterValue: + operator = filterValue.get('operator', 'equals') + value = filterValue.get('value') + else: + operator = 'equals' + value = filterValue + + if value is None or value == '': + continue + + def matchesFilter(item: Dict[str, Any], f: str, op: str, v: Any) -> bool: + itemValue = item.get(f) + if itemValue is None: + return False + + # Convert to string for comparison if needed + itemStr = str(itemValue).lower() + valueStr = str(v).lower() + + if op in ('equals', 'eq'): + return itemStr == valueStr + elif op == 'contains': + return valueStr in itemStr + elif op == 'startsWith': + return itemStr.startswith(valueStr) + elif op == 'endsWith': + return itemStr.endswith(valueStr) + elif op in ('gt', 'gte', 'lt', 'lte'): + try: + itemNum = float(itemValue) + valueNum = float(v) + if op == 'gt': + return itemNum > valueNum + elif op == 'gte': + return itemNum >= valueNum + elif op == 'lt': + return itemNum < valueNum + elif op == 'lte': + return itemNum <= valueNum + except (ValueError, TypeError): + return False + elif op == 'in': + if isinstance(v, list): + return itemStr in [str(x).lower() for x in v] + return False + elif op == 'notIn': + if isinstance(v, list): + return itemStr not in [str(x).lower() for x in v] + return True + return True + + result = [item for item in result if matchesFilter(item, field, operator, value)] + + # Apply sorting + if paginationParams.sort: + for sortField in reversed(paginationParams.sort): + fieldName = sortField.field + ascending = sortField.direction == 'asc' + + def getSortKey(item: Dict[str, Any]): + value = item.get(fieldName) + if value is None: + return (1, '') # Nulls last + if isinstance(value, bool): + return (0, not value if ascending else value) + if isinstance(value, (int, float)): + return (0, value) + return (0, str(value).lower()) + + result = sorted(result, key=getSortKey, reverse=not ascending) + + return result + + router = APIRouter( prefix="/api/users", tags=["Manage Users"], @@ -100,18 +212,24 @@ async def get_users( # Get all users directly from database using UserInDB (the actual database model) from modules.datamodels.datamodelUam import UserInDB allUsers = appInterface.db.getRecordset(UserInDB) - # Convert to User objects, filtering out password hash and database-specific fields - users = [] + # Convert to cleaned dictionaries first for filtering + cleanedUsers = [] for u in allUsers: cleanedUser = {k: v for k, v in u.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} # Ensure roleLabels is always a list if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] - users.append(User(**cleanedUser)) + cleanedUsers.append(cleanedUser) + + # Apply server-side filtering and sorting + filteredUsers = _applyFiltersAndSort(cleanedUsers, paginationParams) + + # Convert to User objects + users = [User(**u) for u in filteredUsers] if paginationParams: - totalItems = len(users) import math + totalItems = len(users) totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 startIdx = (paginationParams.page - 1) * paginationParams.pageSize endIdx = startIdx + paginationParams.pageSize From 04ba89a0e89212d78806041fc5818ab65cb754e0 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 22 Jan 2026 00:23:33 +0100 Subject: [PATCH 11/32] before feature container refactory --- app.py | 20 +- modules/auth/authentication.py | 2 +- modules/auth/tokenManager.py | 2 +- modules/auth/tokenRefreshService.py | 4 +- modules/datamodels/FIELD_NAMES.md | 314 --- modules/datamodels/datamodelChatbot.py | 1061 +++++++++ modules/features/realEstate/mainRealEstate.py | 2 +- .../features/workflow/subAutomationUtils.py | 2 +- ...rfaceDbAppObjects.py => interfaceDbApp.py} | 0 modules/interfaces/interfaceDbChat.py | 1963 +++++++++++++++++ modules/interfaces/interfaceDbChatbot.py | 10 +- ...entObjects.py => interfaceDbManagement.py} | 2 +- modules/routes/routeAdmin.py | 2 +- ...kflow.py => routeAdminAutomationEvents.py} | 8 +- ...routeFeatures.py => routeAdminFeatures.py} | 2 +- ...eRbacExport.py => routeAdminRbacExport.py} | 2 +- modules/routes/routeAdminRbacRoles.py | 2 +- .../{routeRbac.py => routeAdminRbacRules.py} | 2 +- modules/routes/routeDataAutomation.py | 2 +- modules/routes/routeDataConnections.py | 4 +- modules/routes/routeDataFiles.py | 30 +- modules/routes/routeDataMandates.py | 22 +- modules/routes/routeDataPrompts.py | 12 +- modules/routes/routeDataUsers.py | 20 +- ...outeWorkflows.py => routeDataWorkflows.py} | 6 +- modules/routes/routeFeatureChatDynamic.py | 4 +- modules/routes/routeFeatureChatbot.py | 4 +- modules/routes/routeFeatureRealEstate.py | 2 +- modules/routes/routeFeatureTrustee.py | 2 +- modules/routes/routeGdpr.py | 2 +- modules/routes/routeInvitations.py | 2 +- modules/routes/routeMessaging.py | 30 +- modules/routes/routeSecurityAdmin.py | 2 +- modules/routes/routeSecurityGoogle.py | 2 +- modules/routes/routeSecurityLocal.py | 2 +- modules/routes/routeSecurityMsft.py | 2 +- modules/routes/routeSharepoint.py | 2 +- modules/services/__init__.py | 6 +- .../mainServiceExtraction.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 6 +- .../processing/shared/placeholderFactory.py | 6 +- tests/functional/test03_ai_operations.py | 14 +- tests/functional/test04_ai_behavior.py | 6 +- .../test05_workflow_with_documents.py | 8 +- .../test06_workflow_prompt_variations.py | 8 +- .../test09_document_generation_formats.py | 8 +- .../test10_document_generation_formats.py | 8 +- .../test11_code_generation_formats.py | 8 +- tests/integration/options/test_options_api.py | 2 +- 49 files changed, 3172 insertions(+), 462 deletions(-) delete mode 100644 modules/datamodels/FIELD_NAMES.md create mode 100644 modules/datamodels/datamodelChatbot.py rename modules/interfaces/{interfaceDbAppObjects.py => interfaceDbApp.py} (100%) create mode 100644 modules/interfaces/interfaceDbChat.py rename modules/interfaces/{interfaceDbComponentObjects.py => interfaceDbManagement.py} (99%) rename modules/routes/{routeFeatureWorkflow.py => routeAdminAutomationEvents.py} (94%) rename modules/routes/{routeFeatures.py => routeAdminFeatures.py} (99%) rename modules/routes/{routeRbacExport.py => routeAdminRbacExport.py} (99%) rename modules/routes/{routeRbac.py => routeAdminRbacRules.py} (99%) rename modules/routes/{routeWorkflows.py => routeDataWorkflows.py} (99%) diff --git a/app.py b/app.py index 472de2a1..6944b144 100644 --- a/app.py +++ b/app.py @@ -20,7 +20,7 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager from modules.features import featuresLifecycle as featuresLifecycle -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface class DailyRotatingFileHandler(RotatingFileHandler): """ @@ -421,8 +421,8 @@ app.include_router(promptRouter) from modules.routes.routeDataConnections import router as connectionsRouter app.include_router(connectionsRouter) -from modules.routes.routeWorkflows import router as workflowRouter -app.include_router(workflowRouter) +from modules.routes.routeDataWorkflows import router as dataWorkflowsRouter +app.include_router(dataWorkflowsRouter) from modules.routes.routeFeatureChatDynamic import router as chatPlaygroundRouter app.include_router(chatPlaygroundRouter) @@ -451,11 +451,11 @@ app.include_router(sharepointRouter) from modules.routes.routeDataAutomation import router as automationRouter app.include_router(automationRouter) -from modules.routes.routeFeatureWorkflow import router as adminAutomationEventsRouter +from modules.routes.routeAdminAutomationEvents import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) -from modules.routes.routeRbac import router as rbacRouter -app.include_router(rbacRouter) +from modules.routes.routeAdminRbacRules import router as rbacAdminRulesRouter +app.include_router(rbacAdminRulesRouter) from modules.routes.routeOptions import router as optionsRouter app.include_router(optionsRouter) @@ -470,14 +470,14 @@ from modules.routes.routeFeatureTrustee import router as trusteeRouter app.include_router(trusteeRouter) # Phase 8: New Feature Routes -from modules.routes.routeFeatures import router as featuresRouter -app.include_router(featuresRouter) +from modules.routes.routeAdminFeatures import router as featuresAdminRouter +app.include_router(featuresAdminRouter) from modules.routes.routeInvitations import router as invitationsRouter app.include_router(invitationsRouter) -from modules.routes.routeRbacExport import router as rbacExportRouter -app.include_router(rbacExportRouter) +from modules.routes.routeAdminRbacExport import router as rbacAdminExportRouter +app.include_router(rbacAdminExportRouter) from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) diff --git a/modules/auth/authentication.py b/modules/auth/authentication.py index f6cf0f0d..c6eaafad 100644 --- a/modules/auth/authentication.py +++ b/modules/auth/authentication.py @@ -21,7 +21,7 @@ from slowapi.util import get_remote_address from modules.shared.configuration import APP_CONFIG from modules.security.rootAccess import getRootDbAppConnector, getRootUser -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, AuthAuthority, AccessLevel from modules.datamodels.datamodelSecurity import Token from modules.datamodels.datamodelRbac import AccessRule diff --git a/modules/auth/tokenManager.py b/modules/auth/tokenManager.py index 322b2e4e..0fd76092 100644 --- a/modules/auth/tokenManager.py +++ b/modules/auth/tokenManager.py @@ -259,7 +259,7 @@ class TokenManager: try: if interface is None: from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() interface = getInterface(rootUser) diff --git a/modules/auth/tokenRefreshService.py b/modules/auth/tokenRefreshService.py index 2d780364..a4f8e402 100644 --- a/modules/auth/tokenRefreshService.py +++ b/modules/auth/tokenRefreshService.py @@ -159,7 +159,7 @@ class TokenRefreshService: # Get user interface from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() root_interface = getInterface(rootUser) @@ -228,7 +228,7 @@ class TokenRefreshService: # Get user interface from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() root_interface = getInterface(rootUser) diff --git a/modules/datamodels/FIELD_NAMES.md b/modules/datamodels/FIELD_NAMES.md deleted file mode 100644 index e75899ed..00000000 --- a/modules/datamodels/FIELD_NAMES.md +++ /dev/null @@ -1,314 +0,0 @@ -| Field Name | Type Pattern | Models Using It | -|------------|--------------|-----------------| -| `accumulatedJsonString` | str | JsonAccumulationState | -| `action` | str | ActionDefinition | -| `actionId` | str | ChatDocument, ChatMessage | -| `actionList` | List | TaskItem | -| `actionMethod` | str | ChatMessage | -| `actionName` | str | ChatMessage | -| `actionNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `actionObjective` | str | ActionDefinition, TaskContext | -| `actionProgress` | str | ChatMessage | -| `actionResult` | Any | TaskResult | -| `active` | bool | AutomationDefinition | -| `additionalData` | Dict | AiResponseMetadata | -| `aiPrompt` | str | AiProcessParameters | -| `allSections` | List | JsonAccumulationState | -| `apiUrl` | str | AiModel | -| `authenticationAuthority` | AuthAuthority | User | -| `authority` | AuthAuthority | UserConnection, Token | -| `availableConnections` | list | TaskContext | -| `availableDocuments` | str | TaskContext | -| `base64Encoded` | bool | ContentMetadata, FileData | -| `bytesSent` | int | ChatStat, AiCallResponse | -| `bytesReceived` | int | ChatStat, AiCallResponse | -| `classes` | List | CodeContentPromptArgs | -| `colorMode` | str | ContentMetadata | -| `compressContext` | bool | AiCallOptions | -| `compressPrompt` | bool | AiCallOptions | -| `condition` | str | SelectionRule | -| `confidence` | float | ReviewResult | -| `connectedAt` | float | UserConnection | -| `connectionId` | str | Token | -| `connectionReference` | str | ActionDefinition | -| `connectorType` | str | AiModel | -| `content` | str | Prompt, ContentItem, AiResponse, AiCallResponse, FilePreview | -| `contentAnalysis` | Dict | Observation | -| `contentParts` | List | AiCallRequest, AiProcessParameters, SectionPromptArgs, ChapterStructurePromptArgs, CodeContentPromptArgs, CodeStructurePromptArgs | -| `contentSize` | str | ObservationPreview | -| `contentValidation` | Dict | Observation | -| `contents` | List | ChatContentExtracted | -| `context` | Dict, AccessRuleContext | TaskHandover, UnderstandingResult, AccessRule | -| `contextInfo` | str | CodeContentPromptArgs | -| `costPer1kTokensInput` | float | AiModel | -| `costPer1kTokensOutput` | float | AiModel | -| `country` | str | AiCallPromptWebSearch | -| `create` | AccessLevel | AccessRule, UserPermissions | -| `created` | str | ObservationPreview | -| `createdAt` | float | Token | -| `creationDate` | float | FileItem | -| `criteriaProgress` | dict | TaskContext | -| `currentAction` | int | ChatWorkflow | -| `currentRound` | int | ChatWorkflow | -| `currentTask` | int | ChatWorkflow | -| `data` | str | ContentItem, FileData | -| `dataType` | str | TaskStep | -| `delete` | AccessLevel | AccessRule, UserPermissions | -| `deliverable` | Dict | TaskDefinition | -| `delivered_summary` | str | ContinuationContext | -| `dependencies` | List | TaskItem, TaskStep, CodeContentPromptArgs | -| `description` | TextMultilingual | Role | -| `details` | str | AuthEvent | -| `detectedComplexity` | str | RequestContext | -| `displayName` | str | AiModel | -| `documentData` | Any | ActionDocument, DocumentData | -| `documentId` | str | ObservationPreview | -| `documentList` | DocumentReferenceList | ActionDefinition, ExtractContentParameters | -| `documentName` | str | ActionDocument, DocumentData | -| `documentReferences` | List | UnderstandingResult | -| `documents` | List | ChatMessage, ActionResult, AiResponse | -| `documentsCount` | int | Observation | -| `documentsLabel` | str | ChatMessage, DocumentExchange | -| `durationSec` | float | ContentMetadata | -| `email` | EmailStr | User | -| `enabled` | bool | Mandate, User | -| `encoding` | str | FilePreview | -| `engine` | str | ChatStat | -| `error` | str | ContentMetadata, ActionResult, ActionItem, TaskItem, TaskResult | -| `errorCount` | int | ChatStat, AiCallResponse | -| `estimatedComplexity` | str | TaskStep | -| `eventId` | str | AutomationDefinition | -| `eventType` | str | AuthEvent | -| `execAction` | str | ActionItem | -| `execMethod` | str | ActionItem | -| `execParameters` | Dict | ActionItem | -| `execResultLabel` | str | ActionItem | -| `executedActions` | list | TaskContext | -| `executionLogs` | List | AutomationDefinition | -| `expectedDocumentFormats` | List | ActionItem | -| `expectedFormats` | List | ChatWorkflow, TaskStep | -| `expectedOutputFormat` | str | RequestContext | -| `expectedOutputType` | str | RequestContext | -| `expiresAt` | float | UserConnection, Token | -| `externalEmail` | EmailStr | UserConnection | -| `externalId` | str | UserConnection | -| `externalUsername` | str | UserConnection | -| `extractionMethod` | str | AiResponseMetadata | -| `extractionOptions` | Any | TaskDefinition, ExtractContentParameters | -| `failedActions` | list | TaskContext | -| `failurePatterns` | list | TaskContext | -| `feedback` | str | TaskResult, TaskItem | -| `fileHash` | str | FileItem | -| `fileId` | str | ChatDocument | -| `fileName` | str | FileItem, FilePreview, ChatDocument | -| `fileSize` | int | FileItem, ChatDocument | -| `fileType` | str | CodeContentPromptArgs | -| `filename` | str | AiResponseMetadata, CodeContentPromptArgs | -| `finishedAt` | float | TaskItem | -| `fps` | float | ContentMetadata | -| `fullName` | str | User | -| `functionCall` | Callable | AiModel | -| `functions` | List | CodeContentPromptArgs | -| `generationHint` | str | SectionPromptArgs | -| `handoverType` | str | TaskHandover | -| `hashedPassword` | str | UserInDB | -| `height` | int | ContentMetadata | -| `hierarchyContext` | str | JsonContinuationContexts | -| `hierarchyContextForPrompt` | str | JsonContinuationContexts | -| `id` | str | *Most models* | -| `improvements` | List | TaskHandover, TaskContext, ReviewResult | -| `incomplete_part` | str | ContinuationContext | -| `inputDocuments` | List | TaskHandover | -| `instruction` | str | AiCallPromptWebSearch, AiCallPromptWebCrawl | -| `intention` | Dict | UnderstandingResult | -| `ipAddress` | str | AuthEvent | -| `isAccumulationMode` | bool | JsonAccumulationState | -| `isAggregation` | bool | SectionPromptArgs | -| `isAvailable` | bool | AiModel | -| `isRegeneration` | bool | TaskContext | -| `isSystemRole` | bool | Role | -| `isText` | bool | FilePreview | -| `item` | str | AccessRule | -| `jsonParsingSuccess` | bool | JsonContinuationContexts | -| `kpis` | List | JsonAccumulationState | -| `label` | str | ContentItem, AutomationDefinition | -| `language` | str | Mandate, User, SectionPromptArgs, AiCallPromptWebSearch | -| `last_complete_part` | str | ContinuationContext | -| `last_raw_json` | str | ContinuationContext | -| `lastActivity` | float | ChatWorkflow | -| `lastChecked` | float | UserConnection | -| `lastParsedResult` | Dict | JsonAccumulationState | -| `lastUpdated` | str | AiModel | -| `learnings` | List | ActionDefinition, TaskContext | -| `listFileId` | List | UserInputRequest | -| `logs` | List | ChatWorkflow | -| `mandateId` | str | ChatWorkflow, FileItem, Prompt, User, AutomationDefinition, Token | -| `maxCost` | float | SelectionRule, AiCallOptions | -| `maxDepth` | int | AiCallPromptWebCrawl | -| `maxNumberPages` | int | AiCallPromptWebSearch | -| `maxParts` | int | AiCallOptions | -| `maxProcessingTime` | int | AiCallOptions | -| `maxSteps` | int | ChatWorkflow | -| `maxTokens` | int | AiModel | -| `maxWidth` | int | AiCallPromptWebCrawl | -| `message` | str | ChatLog, ChatMessage | -| `messageHistory` | List | TaskHandover | -| `messageId` | str | ChatDocument | -| `messages` | List | ChatWorkflow, AiModelCall | -| `metadata` | ContentMetadata, AiResponseMetadata, Dict | ContentItem, AiResponse, AiModelResponse, CodeContentPromptArgs | -| `metCriteria` | List | ReviewResult | -| `method` | str | ActionSelection | -| `mime` | str | ObservationPreview | -| `mimeType` | str | ContentMetadata, FileItem, FilePreview, ChatDocument, ActionDocument, DocumentData | -| `minContextLength` | int | AiModel, SelectionRule | -| `minQualityRating` | int | SelectionRule | -| `missingOutputs` | List | ReviewResult | -| `model` | AiModel | AiModelCall | -| `modelId` | str | AiModelResponse | -| `modelName` | str | AiCallResponse | -| `modified` | str | ObservationPreview | -| `name` | str | Mandate, Prompt, ActionSelection, AiModel, SelectionRule, ObservationPreview | -| `nextAction` | str | ReviewResult | -| `nextActionGuidance` | Dict | TaskContext | -| `nextActionObjective` | str | ReviewResult | -| `nextActionParameters` | Dict | ReviewResult | -| `notes` | List | Observation | -| `objective` | str | TaskStep, TaskDefinition | -| `operationId` | str | ChatLog | -| `operationType` | OperationTypeEnum, str | OperationTypeRating, AiCallOptions, AiResponseMetadata | -| `operationTypes` | List | AiModel, SelectionRule | -| `options` | AiCallOptions | AiCallRequest, AiModelCall | -| `originalPrompt` | str | RequestContext | -| `outputDocuments` | List | TaskHandover | -| `outputFormat` | str | ChapterStructurePromptArgs | -| `overlapContext` | str | JsonContinuationContexts | -| `overlap_context` | str | ContinuationContext | -| `overview` | str | TaskPlan | -| `pages` | int | ContentMetadata | -| `parameters` | Dict | ActionParameters, UnderstandingResult, ActionDefinition | -| `parametersContext` | str | ActionDefinition, TaskContext | -| `parentId` | str | ChatLog | -| `parentMessageId` | str | ChatMessage | -| `performance` | Dict | ChatLog | -| `placeholders` | List, Dict | PromptBundle, AutomationDefinition | -| `priceUsd` | float | ChatStat, AiCallResponse | -| `previews` | List | Observation | -| `previousActionResults` | list | TaskContext | -| `previousHandover` | TaskHandover | TaskContext | -| `previousResults` | List | TaskHandover, TaskContext, ReviewContext | -| `previousReviewResult` | dict | TaskContext | -| `priority` | PriorityEnum | AiModel, SelectionRule, AiCallOptions | -| `process` | str | ChatStat | -| `processDocumentsIndividually` | bool | AiCallOptions | -| `processingMode` | ProcessingModeEnum | AiModel, AiCallOptions | -| `processingTime` | float | ChatStat, ActionItem, TaskItem, AiCallResponse, AiModelResponse | -| `progress` | float | ChatLog | -| `prompt` | str | UserInputRequest, PromptBundle, AiCallPromptImage | -| `publishedAt` | float | ChatMessage | -| `qualityRating` | int | AiModel | -| `qualityRequirements` | Dict | TaskStep | -| `qualityScore` | float | ReviewResult | -| `quality` | str | AiCallPromptImage | -| `rating` | int | OperationTypeRating | -| `read` | AccessLevel | AccessRule, UserPermissions | -| `reason` | str | Token, ReviewResult | -| `reference` | str | ObservationPreview | -| `requiredDocuments` | List | TaskDefinition | -| `requiresAnalysis` | bool | RequestContext | -| `requiresContentGeneration` | bool | TaskDefinition | -| `requiresDocumentAnalysis` | bool | TaskDefinition | -| `requiresDocuments` | bool | RequestContext | -| `requiresWebResearch` | bool | RequestContext, TaskDefinition | -| `researchDepth` | str | AiCallPromptWebSearch | -| `resetToken` | str | UserInDB | -| `resetTokenExpires` | float | UserInDB | -| `result` | str | ActionItem | -| `resultFormat` | str | AiCallOptions | -| `resultLabel` | str | ActionResult, Observation | -| `resultLabels` | Dict | TaskItem | -| `resultType` | str | AiProcessParameters | -| `retryCount` | int | ActionItem, TaskItem, TaskContext | -| `retryMax` | int | ActionItem, TaskItem | -| `revokedAt` | float | Token | -| `revokedBy` | str | Token | -| `role` | str | ChatMessage | -| `roleLabel` | str | Role, AccessRule | -| `roleLabels` | List | User | -| `rollbackOnFailure` | bool | TaskItem | -| `roundNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `safetyMargin` | float | AiCallOptions | -| `schedule` | str | AutomationDefinition | -| `schemaVersion` | str | AiResponseMetadata | -| `section` | Dict | SectionPromptArgs | -| `section_count` | int | ContinuationContext | -| `sectionIndex` | int | SectionPromptArgs | -| `sequenceNr` | int | ChatMessage | -| `sessionId` | str | Token | -| `size` | int, str | ContentMetadata, FilePreview, AiCallPromptImage, ObservationPreview | -| `snippet` | str | ObservationPreview | -| `sourceDocuments` | List | AiResponseMetadata | -| `sourceJson` | Dict | ActionDocument, DocumentData | -| `sourceTask` | str | TaskHandover | -| `speedRating` | int | AiModel | -| `stage1Selection` | dict | TaskContext | -| `startedAt` | float | ChatWorkflow, TaskItem | -| `stats` | List | ChatWorkflow | -| `status` | str, TokenStatus, TaskStatus, ConnectionStatus | ChatLog, ChatWorkflow, ChatMessage, UserConnection, AutomationDefinition, ActionItem, TaskItem, TaskResult, ReviewResult, Token | -| `style` | str | AiCallPromptImage | -| `success` | bool | ChatMessage, ActionResult, Observation, TaskResult, AuthEvent, AiModelResponse | -| `successCriteria` | list | TaskStep | -| `successfulActions` | list | TaskContext | -| `summary` | str | ChatMessage | -| `summaryAllowed` | bool | PromptPlaceholder | -| `taskActions` | list | ReviewContext | -| `taskId` | str | TaskResult, TaskHandover | -| `taskNumber` | int | ChatLog, ChatDocument, ChatMessage | -| `taskProgress` | str | ChatMessage | -| `tasks` | List | ChatWorkflow, TaskPlan, UnderstandingResult | -| `taskStep` | TaskStep | TaskContext, ReviewContext | -| `temperature` | float | AiModel, AiCallOptions | -| `template` | str | AutomationDefinition | -| `template_structure` | str | ContinuationContext | -| `timestamp` | float | ChatLog, ActionItem, TaskHandover, AuthEvent | -| `title` | str | AiResponseMetadata | -| `tokenAccess` | str | Token | -| `tokenExpiresAt` | float | UserConnection | -| `tokenRefresh` | str | Token | -| `tokensUsed` | Dict | AiModelResponse | -| `tokenStatus` | str | UserConnection | -| `tokenType` | str | Token | -| `totalActions` | int | ChatWorkflow | -| `totalTasks` | int | ChatWorkflow | -| `type` | str | ChatLog | -| `typeGroup` | str | ObservationPreview | -| `unmetCriteria` | List | ReviewResult | -| `update` | AccessLevel | AccessRule, UserPermissions | -| `url` | str | AiCallPromptWebCrawl | -| `userAgent` | str | AuthEvent | -| `userId` | str | UserConnection, Token, AuthEvent | -| `userInput` | str | TaskItem | -| `userLanguage` | str | UserInputRequest, RequestContext | -| `userMessage` | str | ActionItem, TaskStep, ReviewResult, TaskPlan, ActionDefinition | -| `userPrompt` | str | SectionPromptArgs, ChapterStructurePromptArgs, CodeContentPromptArgs, CodeStructurePromptArgs | -| `username` | str | User | -| `validationMetadata` | Dict | ActionDocument | -| `version` | str | AiModel | -| `view` | bool | AccessRule, UserPermissions | -| `weight` | float | SelectionRule | -| `width` | int | ContentMetadata | -| `workflow` | ChatWorkflow | TaskContext | -| `workflowId` | str | ChatStat, ChatLog, ChatMessage, ChatWorkflow, TaskItem, TaskContext, ReviewContext | -| `workflowMode` | WorkflowModeEnum | ChatWorkflow | -| `workflowSummary` | str | TaskHandover | - - - ---- - -Can you adapt following fields to Multilingual Fields (`TextMultilingual`): - -| Field Name | Models | -|------------|--------| -| `description` | Role | (is already) - - diff --git a/modules/datamodels/datamodelChatbot.py b/modules/datamodels/datamodelChatbot.py new file mode 100644 index 00000000..45c5c4eb --- /dev/null +++ b/modules/datamodels/datamodelChatbot.py @@ -0,0 +1,1061 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chat models: ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument.""" + +from typing import List, Dict, Any, Optional +from enum import Enum +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class ChatStat(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this stat belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this stat belongs to" + ) + workflowId: Optional[str] = Field( + None, description="Foreign key to workflow (for workflow stats)" + ) + processingTime: Optional[float] = Field( + None, description="Processing time in seconds" + ) + bytesSent: Optional[int] = Field(None, description="Number of bytes sent") + bytesReceived: Optional[int] = Field(None, description="Number of bytes received") + errorCount: Optional[int] = Field(None, description="Number of errors encountered") + process: Optional[str] = Field(None, description="The process that delivers the stats data (e.g. 'action.outlook.readMails', 'ai.process.document.name')") + engine: Optional[str] = Field(None, description="The engine used (e.g. 'ai.anthropic.35', 'ai.tavily.basic', 'renderer.docx')") + priceUsd: Optional[float] = Field(None, description="Calculated price in USD for the operation") + + +registerModelLabels( + "ChatStat", + {"en": "Chat Statistics", "fr": "Statistiques de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, + "bytesReceived": {"en": "Bytes Received", "fr": "Octets reçus"}, + "errorCount": {"en": "Error Count", "fr": "Nombre d'erreurs"}, + "process": {"en": "Process", "fr": "Processus"}, + "engine": {"en": "Engine", "fr": "Moteur"}, + "priceUsd": {"en": "Price USD", "fr": "Prix USD"}, + }, +) + + +class ChatLog(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this log belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this log belongs to" + ) + workflowId: str = Field(description="Foreign key to workflow") + message: str = Field(description="Log message") + type: str = Field(description="Log type (info, warning, error, etc.)") + timestamp: float = Field( + default_factory=getUtcTimestamp, + description="When the log entry was created (UTC timestamp in seconds)", + ) + status: Optional[str] = Field(None, description="Status of the log entry") + progress: Optional[float] = Field( + None, description="Progress indicator (0.0 to 1.0)" + ) + performance: Optional[Dict[str, Any]] = Field( + None, description="Performance metrics" + ) + parentId: Optional[str] = Field( + None, description="Parent operation ID (operationId of parent operation) for hierarchical display" + ) + operationId: Optional[str] = Field( + None, description="Operation ID to group related log entries" + ) + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + + +registerModelLabels( + "ChatLog", + {"en": "Chat Log", "fr": "Journal de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "message": {"en": "Message", "fr": "Message"}, + "type": {"en": "Type", "fr": "Type"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "status": {"en": "Status", "fr": "Statut"}, + "progress": {"en": "Progress", "fr": "Progression"}, + "performance": {"en": "Performance", "fr": "Performance"}, + }, +) + + +class ChatDocument(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this document belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to" + ) + messageId: str = Field(description="Foreign key to message") + fileId: str = Field(description="Foreign key to file") + fileName: str = Field(description="Name of the file") + fileSize: int = Field(description="Size of the file") + mimeType: str = Field(description="MIME type of the file") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + actionId: Optional[str] = Field( + None, description="ID of the action that created this document" + ) + + +registerModelLabels( + "ChatDocument", + {"en": "Chat Document", "fr": "Document de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "messageId": {"en": "Message ID", "fr": "ID du message"}, + "fileId": {"en": "File ID", "fr": "ID du fichier"}, + "fileName": {"en": "File Name", "fr": "Nom du fichier"}, + "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + }, +) + + +class ContentMetadata(BaseModel): + size: int = Field(description="Content size in bytes") + pages: Optional[int] = Field( + None, description="Number of pages for multi-page content" + ) + error: Optional[str] = Field(None, description="Processing error if any") + width: Optional[int] = Field(None, description="Width in pixels for images/videos") + height: Optional[int] = Field( + None, description="Height in pixels for images/videos" + ) + colorMode: Optional[str] = Field(None, description="Color mode") + fps: Optional[float] = Field(None, description="Frames per second for videos") + durationSec: Optional[float] = Field( + None, description="Duration in seconds for media" + ) + mimeType: str = Field(description="MIME type of the content") + base64Encoded: bool = Field(description="Whether the data is base64 encoded") + + +registerModelLabels( + "ContentMetadata", + {"en": "Content Metadata", "fr": "Métadonnées du contenu"}, + { + "size": {"en": "Size", "fr": "Taille"}, + "pages": {"en": "Pages", "fr": "Pages"}, + "error": {"en": "Error", "fr": "Erreur"}, + "width": {"en": "Width", "fr": "Largeur"}, + "height": {"en": "Height", "fr": "Hauteur"}, + "colorMode": {"en": "Color Mode", "fr": "Mode de couleur"}, + "fps": {"en": "FPS", "fr": "IPS"}, + "durationSec": {"en": "Duration", "fr": "Durée"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "base64Encoded": {"en": "Base64 Encoded", "fr": "Encodé en Base64"}, + }, +) + + +class ContentItem(BaseModel): + label: str = Field(description="Content label") + data: str = Field(description="Extracted text content") + metadata: ContentMetadata = Field(description="Content metadata") + + +registerModelLabels( + "ContentItem", + {"en": "Content Item", "fr": "Élément de contenu"}, + { + "label": {"en": "Label", "fr": "Étiquette"}, + "data": {"en": "Data", "fr": "Données"}, + "metadata": {"en": "Metadata", "fr": "Métadonnées"}, + }, +) + + +class ChatContentExtracted(BaseModel): + id: str = Field(description="Reference to source ChatDocument") + contents: List[ContentItem] = Field( + default_factory=list, description="List of content items" + ) + + +registerModelLabels( + "ChatContentExtracted", + {"en": "Extracted Content", "fr": "Contenu extrait"}, + { + "id": {"en": "Object ID", "fr": "ID de l'objet"}, + "contents": {"en": "Contents", "fr": "Contenus"}, + }, +) + + +class ChatMessage(BaseModel): + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), description="Primary key" + ) + mandateId: str = Field( + description="ID of the mandate this message belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this message belongs to" + ) + workflowId: str = Field(description="Foreign key to workflow") + parentMessageId: Optional[str] = Field( + None, description="Parent message ID for threading" + ) + documents: List[ChatDocument] = Field( + default_factory=list, description="Associated documents" + ) + documentsLabel: Optional[str] = Field( + None, description="Label for the set of documents" + ) + message: Optional[str] = Field(None, description="Message content") + summary: Optional[str] = Field( + None, description="Short summary of this message for planning/history" + ) + role: str = Field(description="Role of the message sender") + status: str = Field(description="Status of the message (first, step, last)") + sequenceNr: int = Field( + description="Sequence number of the message (set automatically)" + ) + publishedAt: float = Field( + default_factory=getUtcTimestamp, + description="When the message was published (UTC timestamp in seconds)", + ) + success: Optional[bool] = Field( + None, description="Whether the message processing was successful" + ) + actionId: Optional[str] = Field( + None, description="ID of the action that produced this message" + ) + actionMethod: Optional[str] = Field( + None, description="Method of the action that produced this message" + ) + actionName: Optional[str] = Field( + None, description="Name of the action that produced this message" + ) + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + taskProgress: Optional[str] = Field( + None, description="Task progress status: pending, running, success, fail, retry" + ) + actionProgress: Optional[str] = Field( + None, description="Action progress status: pending, running, success, fail" + ) + + +registerModelLabels( + "ChatMessage", + {"en": "Chat Message", "fr": "Message de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "message": {"en": "Message", "fr": "Message"}, + "summary": {"en": "Summary", "fr": "Résumé"}, + "role": {"en": "Role", "fr": "Rôle"}, + "status": {"en": "Status", "fr": "Statut"}, + "sequenceNr": {"en": "Sequence Number", "fr": "Numéro de séquence"}, + "publishedAt": {"en": "Published At", "fr": "Publié le"}, + "success": {"en": "Success", "fr": "Succès"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + "actionMethod": {"en": "Action Method", "fr": "Méthode de l'action"}, + "actionName": {"en": "Action Name", "fr": "Nom de l'action"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "taskProgress": {"en": "Task Progress", "fr": "Progression de la tâche"}, + "actionProgress": {"en": "Action Progress", "fr": "Progression de l'action"}, + }, +) + + +class WorkflowModeEnum(str, Enum): + WORKFLOW_DYNAMIC = "Dynamic" + WORKFLOW_AUTOMATION = "Automation" + WORKFLOW_CHATBOT = "Chatbot" + WORKFLOW_REACT = "React" # Legacy mode - kept for backward compatibility + + +registerModelLabels( + "WorkflowModeEnum", + {"en": "Workflow Mode", "fr": "Mode de workflow"}, + { + "WORKFLOW_DYNAMIC": {"en": "Dynamic", "fr": "Dynamique"}, + "WORKFLOW_AUTOMATION": {"en": "Automation", "fr": "Automatisation"}, + "WORKFLOW_CHATBOT": {"en": "Chatbot", "fr": "Chatbot"}, + "WORKFLOW_REACT": {"en": "React (Legacy)", "fr": "React (Hérité)"}, + }, +) + + +class ChatWorkflow(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: str = Field(description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, + {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, + {"value": "stopped", "label": {"en": "Stopped", "fr": "Arrêté"}}, + {"value": "error", "label": {"en": "Error", "fr": "Erreur"}}, + ]}) + name: Optional[str] = Field(None, description="Name of the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) + currentRound: int = Field(default=0, description="Current round number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + currentTask: int = Field(default=0, description="Current task number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + currentAction: int = Field(default=0, description="Current action number", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + totalTasks: int = Field(default=0, description="Total number of tasks in the workflow", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + totalActions: int = Field(default=0, description="Total number of actions in the workflow", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) + lastActivity: float = Field(default_factory=getUtcTimestamp, description="Timestamp of last activity (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + startedAt: float = Field(default_factory=getUtcTimestamp, description="When the workflow started (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + logs: List[ChatLog] = Field(default_factory=list, description="Workflow logs", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + messages: List[ChatMessage] = Field(default_factory=list, description="Messages in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + stats: List[ChatStat] = Field(default_factory=list, description="Workflow statistics list", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + tasks: list = Field(default_factory=list, description="List of tasks in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + workflowMode: WorkflowModeEnum = Field(default=WorkflowModeEnum.WORKFLOW_DYNAMIC, description="Workflow mode selector", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + { + "value": WorkflowModeEnum.WORKFLOW_DYNAMIC.value, + "label": {"en": "Dynamic", "fr": "Dynamique"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_AUTOMATION.value, + "label": {"en": "Automation", "fr": "Automatisation"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_CHATBOT.value, + "label": {"en": "Chatbot", "fr": "Chatbot"}, + }, + { + "value": WorkflowModeEnum.WORKFLOW_REACT.value, + "label": {"en": "React (Legacy)", "fr": "React (Hérité)"}, + }, + ]}) + maxSteps: int = Field(default=10, description="Maximum number of iterations in dynamic mode", json_schema_extra={"frontend_type": "integer", "frontend_readonly": False, "frontend_required": False}) + expectedFormats: Optional[List[str]] = Field(None, description="List of expected file format extensions from user request (e.g., ['xlsx', 'pdf']). Extracted during intent analysis.", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + + # Helper methods for execution state management + def getRoundIndex(self) -> int: + """Get current round index""" + return self.currentRound + + def getTaskIndex(self) -> int: + """Get current task index""" + return self.currentTask + + def getActionIndex(self) -> int: + """Get current action index""" + return self.currentAction + + def incrementRound(self): + """Increment round when new user input received""" + self.currentRound += 1 + self.currentTask = 0 + self.currentAction = 0 + + def incrementTask(self): + """Increment task when starting new task in current round""" + self.currentTask += 1 + self.currentAction = 0 + + def incrementAction(self): + """Increment action when executing new action in current task""" + self.currentAction += 1 + + +registerModelLabels( + "ChatWorkflow", + {"en": "Chat Workflow", "fr": "Flux de travail de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "status": {"en": "Status", "fr": "Statut"}, + "name": {"en": "Name", "fr": "Nom"}, + "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, + "currentTask": {"en": "Current Task", "fr": "Tâche actuelle"}, + "currentAction": {"en": "Current Action", "fr": "Action actuelle"}, + "totalTasks": {"en": "Total Tasks", "fr": "Total des tâches"}, + "totalActions": {"en": "Total Actions", "fr": "Total des actions"}, + "lastActivity": {"en": "Last Activity", "fr": "Dernière activité"}, + "startedAt": {"en": "Started At", "fr": "Démarré le"}, + "logs": {"en": "Logs", "fr": "Journaux"}, + "messages": {"en": "Messages", "fr": "Messages"}, + "stats": {"en": "Statistics", "fr": "Statistiques"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "workflowMode": {"en": "Workflow Mode", "fr": "Mode de workflow"}, + "maxSteps": {"en": "Max Steps", "fr": "Étapes max"}, + "expectedFormats": {"en": "Expected Formats", "fr": "Formats attendus"}, + }, +) + + +class UserInputRequest(BaseModel): + prompt: str = Field(description="Prompt for the user") + listFileId: List[str] = Field(default_factory=list, description="List of file IDs") + userLanguage: str = Field(default="en", description="User's preferred language") + workflowId: Optional[str] = Field(None, description="Optional ID of the workflow to continue") + + +registerModelLabels( + "UserInputRequest", + {"en": "User Input Request", "fr": "Demande de saisie utilisateur"}, + { + "prompt": {"en": "Prompt", "fr": "Invite"}, + "listFileId": {"en": "File IDs", "fr": "IDs des fichiers"}, + "userLanguage": {"en": "User Language", "fr": "Langue de l'utilisateur"}, + }, +) + + +class ActionDocument(BaseModel): + """Clear document structure for action results""" + + documentName: str = Field(description="Name of the document") + documentData: Any = Field(description="Content/data of the document") + mimeType: str = Field(description="MIME type of the document") + sourceJson: Optional[Dict[str, Any]] = Field( + None, + description="Source JSON structure (preserved when rendering to xlsx/docx/pdf)" + ) + validationMetadata: Optional[Dict[str, Any]] = Field( + None, + description="Action-specific metadata for content validation (e.g., email recipients, attachments, SharePoint paths)" + ) + + +registerModelLabels( + "ActionDocument", + {"en": "Action Document", "fr": "Document d'action"}, + { + "documentName": {"en": "Document Name", "fr": "Nom du document"}, + "documentData": {"en": "Document Data", "fr": "Données du document"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + }, +) + + +class ActionResult(BaseModel): + """Clean action result with documents as primary output + + IMPORTANT: Action methods should NOT set resultLabel in their return value. + The resultLabel is managed by the action handler using the action's execResultLabel + from the action plan. This ensures consistent document routing throughout the workflow. + """ + + success: bool = Field(description="Whether execution succeeded") + error: Optional[str] = Field(None, description="Error message if failed") + documents: List[ActionDocument] = Field( + default_factory=list, description="Document outputs" + ) + resultLabel: Optional[str] = Field( + None, + description="Label for document routing (set by action handler, not by action methods)", + ) + + @classmethod + def isSuccess(cls, documents: List[ActionDocument] = None) -> "ActionResult": + return cls(success=True, documents=documents or []) + + @classmethod + def isFailure( + cls, error: str, documents: List[ActionDocument] = None + ) -> "ActionResult": + return cls(success=False, documents=documents or [], error=error) + + +registerModelLabels( + "ActionResult", + {"en": "Action Result", "fr": "Résultat de l'action"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "error": {"en": "Error", "fr": "Erreur"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + }, +) + + +class ActionSelection(BaseModel): + method: str = Field(description="Method to execute (e.g., web, document, ai)") + name: str = Field( + description="Action name within the method (e.g., search, extract)" + ) + + +registerModelLabels( + "ActionSelection", + {"en": "Action Selection", "fr": "Sélection d'action"}, + { + "method": {"en": "Method", "fr": "Méthode"}, + "name": {"en": "Action Name", "fr": "Nom de l'action"}, + }, +) + + +class ActionParameters(BaseModel): + parameters: Dict[str, Any] = Field( + default_factory=dict, description="Parameters to execute the selected action" + ) + + +registerModelLabels( + "ActionParameters", + {"en": "Action Parameters", "fr": "Paramètres d'action"}, + { + "parameters": {"en": "Parameters", "fr": "Paramètres"}, + }, +) + + +class ObservationPreview(BaseModel): + name: str = Field(description="Document name or URL label") + mime: Optional[str] = Field(default=None, description="MIME type or kind (legacy field)") + snippet: Optional[str] = Field(default=None, description="Short snippet or summary") + # Extended metadata fields + mimeType: Optional[str] = Field(default=None, description="MIME type") + size: Optional[str] = Field(default=None, description="File size") + created: Optional[str] = Field(default=None, description="Creation timestamp") + modified: Optional[str] = Field(default=None, description="Modification timestamp") + typeGroup: Optional[str] = Field(default=None, description="Document type group") + documentId: Optional[str] = Field(default=None, description="Document ID") + reference: Optional[str] = Field(default=None, description="Document reference") + contentSize: Optional[str] = Field(default=None, description="Content size indicator") + + +registerModelLabels( + "ObservationPreview", + {"en": "Observation Preview", "fr": "Aperçu d'observation"}, + { + "name": {"en": "Name", "fr": "Nom"}, + "mime": {"en": "MIME", "fr": "MIME"}, + "snippet": {"en": "Snippet", "fr": "Extrait"}, + }, +) + + +class Observation(BaseModel): + success: bool = Field(description="Action execution success flag") + resultLabel: str = Field(description="Deterministic label for produced documents") + documentsCount: int = Field(description="Number of produced documents") + previews: List[ObservationPreview] = Field( + default_factory=list, description="Compact previews of outputs" + ) + notes: List[str] = Field( + default_factory=list, description="Short notes or key facts" + ) + # Extended fields for enhanced validation + contentValidation: Optional[Dict[str, Any]] = Field( + default=None, description="Content validation results" + ) + contentAnalysis: Optional[Dict[str, Any]] = Field( + default=None, description="Content analysis results" + ) + + +registerModelLabels( + "Observation", + {"en": "Observation", "fr": "Observation"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + "documentsCount": {"en": "Documents Count", "fr": "Nombre de documents"}, + "previews": {"en": "Previews", "fr": "Aperçus"}, + "notes": {"en": "Notes", "fr": "Notes"}, + }, +) + + +class TaskStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +registerModelLabels( + "TaskStatus", + {"en": "Task Status", "fr": "Statut de la tâche"}, + { + "PENDING": {"en": "Pending", "fr": "En attente"}, + "RUNNING": {"en": "Running", "fr": "En cours"}, + "COMPLETED": {"en": "Completed", "fr": "Terminé"}, + "FAILED": {"en": "Failed", "fr": "Échec"}, + "CANCELLED": {"en": "Cancelled", "fr": "Annulé"}, + }, +) + + +class DocumentExchange(BaseModel): + documentsLabel: str = Field(description="Label for the set of documents") + documents: List[str] = Field( + default_factory=list, description="List of document references" + ) + + +registerModelLabels( + "DocumentExchange", + {"en": "Document Exchange", "fr": "Échange de documents"}, + { + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "documents": {"en": "Documents", "fr": "Documents"}, + }, +) + + +class ActionItem(BaseModel): + id: str = Field(..., description="Action ID") + execMethod: str = Field(..., description="Method to execute") + execAction: str = Field(..., description="Action to perform") + execParameters: Dict[str, Any] = Field( + default_factory=dict, description="Action parameters" + ) + execResultLabel: Optional[str] = Field( + None, description="Label for the set of result documents" + ) + expectedDocumentFormats: Optional[List[Dict[str, str]]] = Field( + None, description="Expected document formats (optional)" + ) + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Action status") + error: Optional[str] = Field(None, description="Error message if action failed") + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + processingTime: Optional[float] = Field( + None, description="Processing time in seconds" + ) + timestamp: float = Field( + ..., description="When the action was executed (UTC timestamp in seconds)" + ) + result: Optional[str] = Field(None, description="Result of the action") + + def setSuccess(self, result: str = None) -> None: + """Set the action as successful with optional result""" + self.status = TaskStatus.COMPLETED + self.error = None + if result is not None: + self.result = result + + def setError(self, error_message: str) -> None: + """Set the action as failed with error message""" + self.status = TaskStatus.FAILED + self.error = error_message + + +registerModelLabels( + "ActionItem", + {"en": "Task Action", "fr": "Action de tâche"}, + { + "id": {"en": "Action ID", "fr": "ID de l'action"}, + "execMethod": {"en": "Method", "fr": "Méthode"}, + "execAction": {"en": "Action", "fr": "Action"}, + "execParameters": {"en": "Parameters", "fr": "Paramètres"}, + "execResultLabel": {"en": "Result Label", "fr": "Label du résultat"}, + "expectedDocumentFormats": { + "en": "Expected Document Formats", + "fr": "Formats de documents attendus", + }, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "result": {"en": "Result", "fr": "Résultat"}, + }, +) + + +class TaskResult(BaseModel): + taskId: str = Field(..., description="Task ID") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + success: bool = Field(..., description="Whether the task was successful") + feedback: Optional[str] = Field(None, description="Task feedback message") + error: Optional[str] = Field(None, description="Error message if task failed") + + +registerModelLabels( + "TaskResult", + {"en": "Task Result", "fr": "Résultat de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "status": {"en": "Status", "fr": "Statut"}, + "success": {"en": "Success", "fr": "Succès"}, + "feedback": {"en": "Feedback", "fr": "Retour"}, + "error": {"en": "Error", "fr": "Erreur"}, + }, +) + + +class TaskItem(BaseModel): + id: str = Field(..., description="Task ID") + workflowId: str = Field(..., description="Workflow ID") + userInput: str = Field(..., description="User input that triggered the task") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + error: Optional[str] = Field(None, description="Error message if task failed") + startedAt: Optional[float] = Field( + None, description="When the task started (UTC timestamp in seconds)" + ) + finishedAt: Optional[float] = Field( + None, description="When the task finished (UTC timestamp in seconds)" + ) + actionList: List[ActionItem] = Field( + default_factory=list, description="List of actions to execute" + ) + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + rollbackOnFailure: bool = Field( + default=True, description="Whether to rollback on failure" + ) + dependencies: List[str] = Field( + default_factory=list, description="List of task IDs this task depends on" + ) + feedback: Optional[str] = Field(None, description="Task feedback message") + processingTime: Optional[float] = Field( + None, description="Total processing time in seconds" + ) + resultLabels: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="Map of result labels to their values" + ) + + +registerModelLabels( + "TaskItem", + {"en": "Task", "fr": "Tâche"}, + { + "id": {"en": "Task ID", "fr": "ID de la tâche"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "userInput": {"en": "User Input", "fr": "Entrée utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "startedAt": {"en": "Started At", "fr": "Démarré à"}, + "finishedAt": {"en": "Finished At", "fr": "Terminé à"}, + "actionList": {"en": "Actions", "fr": "Actions"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + }, +) + + +class TaskStep(BaseModel): + id: str + objective: str + dependencies: Optional[list[str]] = Field(default_factory=list) + successCriteria: Optional[list[str]] = Field(default_factory=list) + estimatedComplexity: Optional[str] = None + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + # Format details extracted from intent analysis + dataType: Optional[str] = Field( + None, description="Expected data type (text, numbers, documents, etc.)" + ) + expectedFormats: Optional[List[str]] = Field( + None, description="Expected output file format extensions (e.g., ['docx', 'pdf', 'xlsx']). Use actual file extensions, not conceptual terms." + ) + qualityRequirements: Optional[Dict[str, Any]] = Field( + None, description="Quality requirements and constraints" + ) + + +registerModelLabels( + "TaskStep", + {"en": "Task Step", "fr": "Étape de tâche"}, + { + "id": {"en": "ID", "fr": "ID"}, + "objective": {"en": "Objective", "fr": "Objectif"}, + "dependencies": {"en": "Dependencies", "fr": "Dépendances"}, + "successCriteria": {"en": "Success Criteria", "fr": "Critères de succès"}, + "estimatedComplexity": { + "en": "Estimated Complexity", + "fr": "Complexité estimée", + }, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + "expectedFormats": {"en": "Expected Formats", "fr": "Formats attendus"}, + }, +) + + +class TaskHandover(BaseModel): + taskId: str = Field(description="Target task ID") + sourceTask: Optional[str] = Field(None, description="Source task ID") + inputDocuments: List[DocumentExchange] = Field( + default_factory=list, description="Available input documents" + ) + outputDocuments: List[DocumentExchange] = Field( + default_factory=list, description="Produced output documents" + ) + context: Dict[str, Any] = Field(default_factory=dict, description="Task context") + previousResults: List[str] = Field( + default_factory=list, description="Previous result summaries" + ) + improvements: List[str] = Field( + default_factory=list, description="Improvement suggestions" + ) + workflowSummary: Optional[str] = Field( + None, description="Summarized workflow context" + ) + messageHistory: List[str] = Field( + default_factory=list, description="Key message summaries" + ) + timestamp: float = Field( + ..., description="When the handover was created (UTC timestamp in seconds)" + ) + handoverType: str = Field( + default="task", description="Type of handover: task, phase, or workflow" + ) + + +registerModelLabels( + "TaskHandover", + {"en": "Task Handover", "fr": "Transfert de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "sourceTask": {"en": "Source Task", "fr": "Tâche source"}, + "inputDocuments": {"en": "Input Documents", "fr": "Documents d'entrée"}, + "outputDocuments": {"en": "Output Documents", "fr": "Documents de sortie"}, + "context": {"en": "Context", "fr": "Contexte"}, + "previousResults": {"en": "Previous Results", "fr": "Résultats précédents"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "workflowSummary": {"en": "Workflow Summary", "fr": "Résumé du workflow"}, + "messageHistory": {"en": "Message History", "fr": "Historique des messages"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "handoverType": {"en": "Handover Type", "fr": "Type de transfert"}, + }, +) + + +class TaskContext(BaseModel): + taskStep: TaskStep + workflow: Optional[ChatWorkflow] = None + workflowId: Optional[str] = None + availableDocuments: Optional[str] = "No documents available" + availableConnections: Optional[list[str]] = Field(default_factory=list) + previousResults: Optional[list[str]] = Field(default_factory=list) + previousHandover: Optional[TaskHandover] = None + improvements: Optional[list[str]] = Field(default_factory=list) + retryCount: Optional[int] = 0 + previousActionResults: Optional[list] = Field(default_factory=list) + previousReviewResult: Optional[dict] = None + isRegeneration: Optional[bool] = False + failurePatterns: Optional[list[str]] = Field(default_factory=list) + failedActions: Optional[list] = Field(default_factory=list) + successfulActions: Optional[list] = Field(default_factory=list) + executedActions: Optional[list] = Field(default_factory=list, description="List of executed actions with action name, parameters, and step number") + criteriaProgress: Optional[dict] = None + + # Stage 2 context fields (NEW) + actionObjective: Optional[str] = Field(None, description="Objective for current action") + parametersContext: Optional[str] = Field(None, description="Context for parameter generation") + learnings: Optional[list[str]] = Field(default_factory=list, description="Learnings from previous actions") + stage1Selection: Optional[dict] = Field(None, description="Stage 1 selection data") + nextActionGuidance: Optional[Dict[str, Any]] = Field(None, description="Guidance for the next action from previous refinement") + + def updateFromSelection(self, selection: Any): + """Update context from Stage 1 selection + + Args: + selection: ActionDefinition instance from Stage 1 + """ + from modules.datamodels.datamodelWorkflow import ActionDefinition + + if isinstance(selection, ActionDefinition): + self.actionObjective = selection.actionObjective + self.parametersContext = selection.parametersContext + self.learnings = selection.learnings if selection.learnings else [] + self.stage1Selection = selection.model_dump() + + def getDocumentReferences(self) -> List[str]: + docs = [] + if self.previousHandover: + for doc_exchange in self.previousHandover.inputDocuments: + docs.extend(doc_exchange.documents) + return list(set(docs)) + + def addImprovement(self, improvement: str) -> None: + if improvement not in (self.improvements or []): + if self.improvements is None: + self.improvements = [] + self.improvements.append(improvement) + + +class ReviewContext(BaseModel): + taskStep: TaskStep + taskActions: Optional[list] = Field(default_factory=list) + actionResults: Optional[list] = Field(default_factory=list) + stepResult: Optional[dict] = Field(default_factory=dict) + workflowId: Optional[str] = None + previousResults: Optional[list[str]] = Field(default_factory=list) + + +class ReviewResult(BaseModel): + status: str + reason: Optional[str] = None + improvements: Optional[list[str]] = Field(default_factory=list) + qualityScore: Optional[float] = Field(default=5.0, description="Quality score (0-10)") + missingOutputs: Optional[list[str]] = Field(default_factory=list) + metCriteria: Optional[list[str]] = Field(default_factory=list) + unmetCriteria: Optional[list[str]] = Field(default_factory=list) + confidence: Optional[float] = 0.5 + userMessage: Optional[str] = Field( + None, description="User-friendly message in user's language" + ) + # NEW: Concrete next action guidance (when status is "continue") + nextAction: Optional[str] = Field( + None, description="Specific action to execute next (e.g., 'ai.convert', 'ai.process', 'ai.reformat')" + ) + nextActionParameters: Optional[Dict[str, Any]] = Field( + None, description="Parameters for the next action (e.g., {'fromFormat': 'json', 'toFormat': 'csv'})" + ) + nextActionObjective: Optional[str] = Field( + None, description="What this specific action will achieve" + ) + + +registerModelLabels( + "ReviewResult", + {"en": "Review Result", "fr": "Résultat de l'évaluation"}, + { + "status": {"en": "Status", "fr": "Statut"}, + "reason": {"en": "Reason", "fr": "Raison"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "qualityScore": {"en": "Quality Score", "fr": "Score de qualité"}, + "missingOutputs": {"en": "Missing Outputs", "fr": "Sorties manquantes"}, + "metCriteria": {"en": "Met Criteria", "fr": "Critères respectés"}, + "unmetCriteria": {"en": "Unmet Criteria", "fr": "Critères non respectés"}, + "confidence": {"en": "Confidence", "fr": "Confiance"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + + +class TaskPlan(BaseModel): + overview: str + tasks: list[TaskStep] + userMessage: Optional[str] = Field( + None, description="Overall user-friendly message for the task plan" + ) + + +registerModelLabels( + "TaskPlan", + {"en": "Task Plan", "fr": "Plan de tâches"}, + { + "overview": {"en": "Overview", "fr": "Aperçu"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + +# Forward references resolved automatically since ChatWorkflow is defined above + + +class PromptPlaceholder(BaseModel): + label: str + content: str + summaryAllowed: bool = Field( + default=False, + description="Whether host may summarize content before sending to AI", + ) + + +registerModelLabels( + "PromptPlaceholder", + {"en": "Prompt Placeholder", "fr": "Espace réservé d'invite"}, + { + "label": {"en": "Label", "fr": "Libellé"}, + "content": {"en": "Content", "fr": "Contenu"}, + "summaryAllowed": {"en": "Summary Allowed", "fr": "Résumé autorisé"}, + }, +) + + +class PromptBundle(BaseModel): + prompt: str + placeholders: List[PromptPlaceholder] = Field(default_factory=list) + + +registerModelLabels( + "PromptBundle", + {"en": "Prompt Bundle", "fr": "Lot d'invite"}, + { + "prompt": {"en": "Prompt", "fr": "Invite"}, + "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, + }, +) + + +class AutomationDefinition(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) + schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ + {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, + {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, + {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} + ]}) + template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) + placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) + active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) + eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + + +registerModelLabels( + "AutomationDefinition", + {"en": "Automation Definition", "fr": "Définition d'automatisation"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "label": {"en": "Label", "fr": "Libellé"}, + "schedule": {"en": "Schedule", "fr": "Planification"}, + "template": {"en": "Template", "fr": "Modèle"}, + "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, + "active": {"en": "Active", "fr": "Actif"}, + "eventId": {"en": "Event ID", "fr": "ID de l'événement"}, + "status": {"en": "Status", "fr": "Statut"}, + "executionLogs": {"en": "Execution Logs", "fr": "Journaux d'exécution"}, + }, +) diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 0d386aac..bc729e0d 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -23,7 +23,7 @@ from modules.datamodels.datamodelRealEstate import ( Land, ) from modules.services import getInterface as getServices -from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector logger = logging.getLogger(__name__) diff --git a/modules/features/workflow/subAutomationUtils.py b/modules/features/workflow/subAutomationUtils.py index 906c9caa..97d28719 100644 --- a/modules/features/workflow/subAutomationUtils.py +++ b/modules/features/workflow/subAutomationUtils.py @@ -3,7 +3,7 @@ """ Utility functions for automation feature. -Moved from interfaces/interfaceDbChatbot.py. +Moved from interfaces/interfaceDbChat.py. """ import json diff --git a/modules/interfaces/interfaceDbAppObjects.py b/modules/interfaces/interfaceDbApp.py similarity index 100% rename from modules/interfaces/interfaceDbAppObjects.py rename to modules/interfaces/interfaceDbApp.py diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py new file mode 100644 index 00000000..d171c2ca --- /dev/null +++ b/modules/interfaces/interfaceDbChat.py @@ -0,0 +1,1963 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Interface to LucyDOM database and AI Connectors. +Uses the JSON connector for data access with added language support. +""" + +import logging +import uuid +import math +from typing import Dict, Any, List, Optional, Union + +import asyncio + +from modules.security.rbac import RbacClass +from modules.datamodels.datamodelRbac import AccessRuleContext +from modules.datamodels.datamodelUam import AccessLevel + +from modules.datamodels.datamodelChat import ( + ChatDocument, + ChatStat, + ChatLog, + ChatMessage, + ChatWorkflow, + WorkflowModeEnum, + AutomationDefinition, + UserInputRequest +) +import json +from modules.datamodels.datamodelUam import User + +# DYNAMIC PART: Connectors to the Interface +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC + +# Basic Configurations +from modules.shared.configuration import APP_CONFIG +logger = logging.getLogger(__name__) + +# Singleton factory for Chat instances +_chatInterfaces = {} + + +def storeDebugMessageAndDocuments(message, currentUser) -> None: + """ + Store message and documents (metadata and file bytes) for debugging purposes. + Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ + - message.json, message_text.txt + - document_###_metadata.json + - document_###_ (actual file bytes) + + Args: + message: ChatMessage object to store + currentUser: Current user for component interface access + """ + try: + import os + from datetime import datetime, UTC + from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir + from modules.interfaces.interfaceDbManagement import getInterface + + # Create base debug directory (use base debug dir, not prompts subdirectory) + baseDebugDir = _getBaseDebugDir() + debug_root = os.path.join(baseDebugDir, 'messages') + _ensureDir(debug_root) + + # Generate timestamp + timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] + + # Create message folder name: m_round_task_action_timestamp + # Use actual values from message, not defaults + round_str = str(message.roundNumber) if message.roundNumber is not None else "0" + task_str = str(message.taskNumber) if message.taskNumber is not None else "0" + action_str = str(message.actionNumber) if message.actionNumber is not None else "0" + message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}" + + message_path = os.path.join(debug_root, message_folder) + os.makedirs(message_path, exist_ok=True) + + # Store message data - use dict() instead of model_dump() for compatibility + message_file = os.path.join(message_path, "message.json") + with open(message_file, "w", encoding="utf-8") as f: + # Convert message to dict manually to avoid model_dump() issues + message_dict = { + "id": message.id, + "workflowId": message.workflowId, + "parentMessageId": message.parentMessageId, + "message": message.message, + "role": message.role, + "status": message.status, + "sequenceNr": message.sequenceNr, + "publishedAt": message.publishedAt, + "roundNumber": message.roundNumber, + "taskNumber": message.taskNumber, + "actionNumber": message.actionNumber, + "documentsLabel": message.documentsLabel, + "actionId": message.actionId, + "actionMethod": message.actionMethod, + "actionName": message.actionName, + "success": message.success, + "documents": [] + } + json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str) + + # Store message content as text + if message.message: + message_text_file = os.path.join(message_path, "message_text.txt") + with open(message_text_file, "w", encoding="utf-8") as f: + f.write(str(message.message)) + + # Store documents if provided + if message.documents and len(message.documents) > 0: + # Group documents by documentsLabel + documents_by_label = {} + for doc in message.documents: + label = message.documentsLabel or 'default' + if label not in documents_by_label: + documents_by_label[label] = [] + documents_by_label[label].append(doc) + + # Create subfolder for each document label + for label, docs in documents_by_label.items(): + # Sanitize label for filesystem + safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip() + safe_label = safe_label.replace(' ', '_') + if not safe_label: + safe_label = "default" + + label_folder = os.path.join(message_path, safe_label) + _ensureDir(label_folder) + + # Store each document + for i, doc in enumerate(docs): + # Create document metadata file + doc_meta = { + "id": doc.id, + "messageId": doc.messageId, + "fileId": doc.fileId, + "fileName": doc.fileName, + "fileSize": doc.fileSize, + "mimeType": doc.mimeType, + "roundNumber": doc.roundNumber, + "taskNumber": doc.taskNumber, + "actionNumber": doc.actionNumber, + "actionId": doc.actionId + } + + doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json") + with open(doc_meta_file, "w", encoding="utf-8") as f: + json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str) + + # Also store the actual file bytes next to metadata for debugging + try: + componentInterface = getInterface(currentUser) + file_bytes = componentInterface.getFileData(doc.fileId) + if file_bytes: + # Build a safe filename preserving original name + safe_name = doc.fileName or f"document_{i+1:03d}" + # Avoid path traversal + safe_name = os.path.basename(safe_name) + doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name) + with open(doc_file_path, "wb") as df: + df.write(file_bytes) + else: + pass + except Exception as e: + pass + + except Exception as e: + # Silent fail - don't break main flow + pass + +class ChatObjects: + """ + Interface to Chat database and AI Connectors. + Uses the JSON connector for data access with added language support. + """ + + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """Initializes the Chat Interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) + """ + # Initialize variables + self.currentUser = currentUser # Store User object directly + self.userId = currentUser.id if currentUser else None + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + self.rbac = None # RBAC interface + + # Initialize services + self._initializeServices() + + # Initialize database + self._initializeDatabase() + + # Set user context if provided + if currentUser: + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) + + # ===== Generic Utility Methods ===== + + def _isObjectField(self, fieldType) -> bool: + """Check if a field type represents a complex object (not a simple type).""" + # Simple scalar types + if fieldType in (str, int, float, bool, type(None)): + return False + + # Everything else is an object + return True + + def _separateObjectFields(self, model_class, data: Dict[str, Any]) -> tuple[Dict[str, Any], Dict[str, Any]]: + """Separate simple fields from object fields based on Pydantic model structure.""" + simpleFields = {} + objectFields = {} + + # Get field information from the Pydantic model + modelFields = model_class.model_fields + + for fieldName, value in data.items(): + # Check if this field should be stored as JSONB in the database + if fieldName in modelFields: + fieldInfo = modelFields[fieldName] + # Pydantic v2 only + fieldType = fieldInfo.annotation + + # Always route relational/object fields to object_fields for separate handling + # These fields are stored in separate normalized tables, not as JSONB + if fieldName in ['documents', 'stats', 'logs', 'messages']: + objectFields[fieldName] = value + continue + + # Check if this is a JSONB field (Dict, List, or complex types) + # Purely type-based detection - no hardcoded field names + if (fieldType == dict or + fieldType == list or + (hasattr(fieldType, '__origin__') and fieldType.__origin__ in (dict, list))): + # Store as JSONB - include in simple_fields for database storage + simpleFields[fieldName] = value + elif isinstance(value, (str, int, float, bool, type(None))): + # Simple scalar types + simpleFields[fieldName] = value + else: + # Complex objects that should be filtered out + objectFields[fieldName] = value + else: + # Field not in model - treat as scalar if simple, otherwise filter out + # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector + if fieldName.startswith("_"): + # Metadata fields should be passed through to connector + simpleFields[fieldName] = value + elif isinstance(value, (str, int, float, bool, type(None))): + simpleFields[fieldName] = value + else: + objectFields[fieldName] = value + + return simpleFields, objectFields + + def _initializeServices(self): + pass + + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """Sets the user context for the interface. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) + """ + self.currentUser = currentUser # Store User object directly + self.userId = currentUser.id + # Use mandateId from parameter (Request-Context), not from user object + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + + if not self.userId: + raise ValueError("Invalid user context: id is required") + + # mandateId can be None for sysadmins performing cross-mandate operations + if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): + raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + + # Add language settings + self.userLanguage = currentUser.language # Default user language + + # Initialize RBAC interface + if not self.currentUser: + raise ValueError("User context is required for RBAC") + # Get DbApp connection for RBAC AccessRule queries + from modules.security.rootAccess import getRootDbAppConnector + dbApp = getRootDbAppConnector() + self.rbac = RbacClass(self.db, dbApp=dbApp) + + # Update database context + self.db.updateContext(self.userId) + + def __del__(self): + """Cleanup method to close database connection.""" + if hasattr(self, 'db') and self.db is not None: + try: + self.db.close() + except Exception as e: + logger.error(f"Error closing database connection: {e}") + + + def _initializeDatabase(self): + """Initializes the database connection directly.""" + try: + # Get configuration values with defaults + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_chat" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + # Create database connector directly + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId + ) + + # Initialize database system + self.db.initDbSystem() + + logger.info("Database initialized successfully") + except Exception as e: + logger.error(f"Failed to initialize database: {str(e)}") + raise + + def _initRecords(self): + """Initializes standard records in the database if they don't exist.""" + pass + + + def checkRbacPermission( + self, + modelClass: type, + operation: str, + recordId: Optional[str] = None + ) -> bool: + """ + Check RBAC permission for a specific operation on a table. + + Args: + modelClass: Pydantic model class for the table + operation: Operation to check ('create', 'update', 'delete', 'read') + recordId: Optional record ID for specific record check + + Returns: + Boolean indicating permission + """ + if not self.rbac or not self.currentUser: + return False + + tableName = modelClass.__name__ + permissions = self.rbac.getUserPermissions( + self.currentUser, + AccessRuleContext.DATA, + tableName + ) + + if operation == "create": + return permissions.create != AccessLevel.NONE + elif operation == "update": + return permissions.update != AccessLevel.NONE + elif operation == "delete": + return permissions.delete != AccessLevel.NONE + elif operation == "read": + return permissions.read != AccessLevel.NONE + else: + return False + + def _applyFilters(self, records: List[Dict[str, Any]], filters: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Apply filter criteria to records. + + Supports: + - General search: {"search": "text"} - searches across all text fields + - Field-specific filters: + - Simple: {"status": "running"} - equals match + - With operator: {"status": {"operator": "equals", "value": "running"}} + - Operators: equals, contains, gt, gte, lt, lte, in, notIn, startsWith, endsWith + + Args: + records: List of record dictionaries to filter + filters: Filter criteria dictionary + + Returns: + Filtered list of records + """ + if not filters or not records: + return records + + filtered = [] + + for record in records: + matches = True + + # Handle general search across text fields + if "search" in filters: + search_term = str(filters["search"]).lower() + if search_term: + # Search in all string fields + found = False + for key, value in record.items(): + if isinstance(value, str) and search_term in value.lower(): + found = True + break + elif isinstance(value, (int, float)) and search_term in str(value): + found = True + break + if not found: + matches = False + + # Handle field-specific filters + for field_name, filter_value in filters.items(): + if field_name == "search": + continue # Already handled above + + if field_name not in record: + matches = False + break + + record_value = record.get(field_name) + + # Handle simple value (equals operator) + if not isinstance(filter_value, dict): + if record_value != filter_value: + matches = False + break + continue + + # Handle filter with operator + operator = filter_value.get("operator", "equals") + filter_val = filter_value.get("value") + + if operator in ["equals", "eq"]: + if record_value != filter_val: + matches = False + break + + elif operator == "contains": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if filter_str not in record_str: + matches = False + break + + elif operator == "startsWith": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if not record_str.startswith(filter_str): + matches = False + break + + elif operator == "endsWith": + record_str = str(record_value).lower() if record_value is not None else "" + filter_str = str(filter_val).lower() if filter_val is not None else "" + if not record_str.endswith(filter_str): + matches = False + break + + elif operator == "gt": + try: + record_num = float(record_value) if record_value is not None else float('-inf') + filter_num = float(filter_val) if filter_val is not None else float('-inf') + if record_num <= filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "gte": + try: + record_num = float(record_value) if record_value is not None else float('-inf') + filter_num = float(filter_val) if filter_val is not None else float('-inf') + if record_num < filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "lt": + try: + record_num = float(record_value) if record_value is not None else float('inf') + filter_num = float(filter_val) if filter_val is not None else float('inf') + if record_num >= filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "lte": + try: + record_num = float(record_value) if record_value is not None else float('inf') + filter_num = float(filter_val) if filter_val is not None else float('inf') + if record_num > filter_num: + matches = False + break + except (ValueError, TypeError): + matches = False + break + + elif operator == "in": + if not isinstance(filter_val, list): + filter_val = [filter_val] + if record_value not in filter_val: + matches = False + break + + elif operator == "notIn": + if not isinstance(filter_val, list): + filter_val = [filter_val] + if record_value in filter_val: + matches = False + break + + else: + # Unknown operator - default to equals + if record_value != filter_val: + matches = False + break + + if matches: + filtered.append(record) + + return filtered + + def _applySorting(self, records: List[Dict[str, Any]], sortFields: List[Any]) -> List[Dict[str, Any]]: + """Apply multi-level sorting to records using stable sort (sorts from least to most significant field).""" + if not sortFields: + return records + + # Start with a copy to avoid modifying original + sortedRecords = list(records) + + # Sort from least significant to most significant field (reverse order) + # Python's sort is stable, so this creates proper multi-level sorting + for sortField in reversed(sortFields): + # Handle both dict and object formats + if isinstance(sortField, dict): + fieldName = sortField.get("field") + direction = sortField.get("direction", "asc") + else: + fieldName = getattr(sortField, "field", None) + direction = getattr(sortField, "direction", "asc") + + if not fieldName: + continue + + isDesc = (direction == "desc") + + def sortKey(record): + value = record.get(fieldName) + # Handle None values - place them at the end for both directions + if value is None: + # Use a special value that sorts last + return (1, "") # (is_none_flag, empty_value) - sorts after (0, ...) + else: + # Return tuple with type indicator for proper comparison + if isinstance(value, (int, float)): + return (0, value) + elif isinstance(value, str): + return (0, value) + elif isinstance(value, bool): + return (0, value) + else: + return (0, str(value)) + + # Sort with reverse parameter for descending + sortedRecords.sort(key=sortKey, reverse=isDesc) + + return sortedRecords + + # Utilities + + def getInitialId(self, model_class: type) -> Optional[str]: + """Returns the initial ID for a table.""" + return self.db.getInitialId(model_class) + + + + # Workflow methods + + def getWorkflows(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + """ + Returns workflows based on user access level. + Supports optional pagination, sorting, and filtering. + + Args: + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[Dict[str, Any]] + If pagination is provided: PaginatedResult with items and metadata + """ + # Use RBAC filtering with featureInstanceId for instance-level isolation + filteredWorkflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId + ) + + # If no pagination requested, return all items (no sorting - frontend handles it) + if pagination is None: + return filteredWorkflows + + # Apply filtering (if filters provided) + if pagination.filters: + filteredWorkflows = self._applyFilters(filteredWorkflows, pagination.filters) + + # Apply sorting (in order of sortFields) - only if provided by frontend + if pagination.sort: + filteredWorkflows = self._applySorting(filteredWorkflows, pagination.sort) + + # Count total items after filters + totalItems = len(filteredWorkflows) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedWorkflows = filteredWorkflows[startIdx:endIdx] + + return PaginatedResult( + items=pagedWorkflows, + totalItems=totalItems, + totalPages=totalPages + ) + + def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: + """Returns a workflow by ID if user has access.""" + # Use RBAC filtering with featureInstanceId for instance-level isolation + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId + ) + + if not workflows: + return None + + workflow = workflows[0] + try: + # Load related data from normalized tables + logs = self.getLogs(workflowId) + messages = self.getMessages(workflowId) + stats = self.getStats(workflowId) + + # Validate workflow data against ChatWorkflow model + return ChatWorkflow( + id=workflow["id"], + status=workflow.get("status", "running"), + name=workflow.get("name"), + currentRound=workflow.get("currentRound", 0) or 0, + currentTask=workflow.get("currentTask", 0) or 0, + currentAction=workflow.get("currentAction", 0) or 0, + totalTasks=workflow.get("totalTasks", 0) or 0, + totalActions=workflow.get("totalActions", 0) or 0, + lastActivity=workflow.get("lastActivity", getUtcTimestamp()), + startedAt=workflow.get("startedAt", getUtcTimestamp()), + logs=logs, + messages=messages, + stats=stats, + mandateId=workflow.get("mandateId", self.mandateId) + ) + except Exception as e: + logger.error(f"Error validating workflow data: {str(e)}") + return None + + def createWorkflow(self, workflowData: Dict[str, Any]) -> ChatWorkflow: + """Creates a new workflow if user has permission.""" + if not self.checkRbacPermission(ChatWorkflow, "create"): + raise PermissionError("No permission to create workflows") + + # Set timestamp if not present + currentTime = getUtcTimestamp() + if "startedAt" not in workflowData: + workflowData["startedAt"] = currentTime + + if "lastActivity" not in workflowData: + workflowData["lastActivity"] = currentTime + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in workflowData or not workflowData["mandateId"]: + workflowData["mandateId"] = self.mandateId + if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: + workflowData["featureInstanceId"] = self.featureInstanceId + + # Use generic field separation based on ChatWorkflow model + simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) + + # Create workflow in database + created = self.db.recordCreate(ChatWorkflow, simpleFields) + + + # Convert to ChatWorkflow model (empty related data for new workflow) + return ChatWorkflow( + id=created["id"], + status=created.get("status", "running"), + name=created.get("name"), + currentRound=created.get("currentRound", 0) or 0, + currentTask=created.get("currentTask", 0) or 0, + currentAction=created.get("currentAction", 0) or 0, + totalTasks=created.get("totalTasks", 0) or 0, + totalActions=created.get("totalActions", 0) or 0, + lastActivity=created.get("lastActivity", currentTime), + startedAt=created.get("startedAt", currentTime), + logs=[], + messages=[], + stats=[], + mandateId=created.get("mandateId", self.mandateId), + workflowMode=created["workflowMode"], + maxSteps=created.get("maxSteps", 1) + ) + + def updateWorkflow(self, workflowId: str, workflowData: Dict[str, Any]) -> ChatWorkflow: + """Updates a workflow if user has access.""" + # Check if the workflow exists and user has access + workflow = self.getWorkflow(workflowId) + if not workflow: + return None + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to update workflow {workflowId}") + + # Use generic field separation based on ChatWorkflow model + simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) + + # Set update time for main workflow + simpleFields["lastActivity"] = getUtcTimestamp() + + # Update main workflow in database + updated = self.db.recordModify(ChatWorkflow, workflowId, simpleFields) + + # Removed cascade writes for logs/messages/stats during workflow update. + # CUD for child entities must be executed via dedicated service methods. + + # Load fresh data from normalized tables + logs = self.getLogs(workflowId) + messages = self.getMessages(workflowId) + stats = self.getStats(workflowId) + + # Convert to ChatWorkflow model + return ChatWorkflow( + id=updated["id"], + status=updated.get("status", workflow.status), + name=updated.get("name", workflow.name), + currentRound=updated.get("currentRound", workflow.currentRound), + currentTask=updated.get("currentTask", workflow.currentTask), + currentAction=updated.get("currentAction", workflow.currentAction), + totalTasks=updated.get("totalTasks", workflow.totalTasks), + totalActions=updated.get("totalActions", workflow.totalActions), + lastActivity=updated.get("lastActivity", workflow.lastActivity), + startedAt=updated.get("startedAt", workflow.startedAt), + logs=logs, + messages=messages, + stats=stats, + mandateId=updated.get("mandateId", workflow.mandateId) + ) + + def deleteWorkflow(self, workflowId: str) -> bool: + """Deletes a workflow and all related data if user has access.""" + try: + # Check if the workflow exists and user has access + workflow = self.getWorkflow(workflowId) + if not workflow: + return False + + if not self.checkRbacPermission(ChatWorkflow, "delete", workflowId): + raise PermissionError(f"No permission to delete workflow {workflowId}") + + # CASCADE DELETE: Delete all related data first + + # 1. Delete all workflow messages and their related data + messages = self.getMessages(workflowId) + for message in messages: + messageId = message.id + if messageId: + # Delete message stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # Delete message documents (but NOT the files!) + existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + for doc in existing_docs: + self.db.recordDelete(ChatDocument, doc["id"]) + + # Delete the message itself + self.db.recordDelete(ChatMessage, messageId) + + # 2. Delete workflow stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # 3. Delete workflow logs + existing_logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + for log in existing_logs: + self.db.recordDelete(ChatLog, log["id"]) + + # 4. Finally delete the workflow itself + success = self.db.recordDelete(ChatWorkflow, workflowId) + + return success + + except Exception as e: + logger.error(f"Error deleting workflow {workflowId}: {str(e)}") + return False + + + # Message methods + + def getMessages(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatMessage], PaginatedResult]: + """ + Returns messages for a workflow if user has access to the workflow. + Supports optional pagination, sorting, and filtering. + + Args: + workflowId: The workflow ID to get messages for + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[ChatMessage] + If pagination is provided: PaginatedResult with items and metadata + """ + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) + + # Get messages for this workflow from normalized table + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + + # Convert raw messages to dict format for sorting/filtering + messageDicts = [] + for msg in messages: + messageDicts.append({ + "id": msg.get("id"), + "workflowId": msg.get("workflowId"), + "parentMessageId": msg.get("parentMessageId"), + "documentsLabel": msg.get("documentsLabel"), + "message": msg.get("message"), + "role": msg.get("role", "assistant"), + "status": msg.get("status", "step"), + "sequenceNr": msg.get("sequenceNr", 0), + "publishedAt": msg.get("publishedAt", msg.get("timestamp", getUtcTimestamp())), + "success": msg.get("success"), + "actionId": msg.get("actionId"), + "actionMethod": msg.get("actionMethod"), + "actionName": msg.get("actionName"), + "roundNumber": msg.get("roundNumber"), + "taskNumber": msg.get("taskNumber"), + "actionNumber": msg.get("actionNumber"), + "taskProgress": msg.get("taskProgress"), + "actionProgress": msg.get("actionProgress") + }) + + # Apply default sorting by publishedAt if no sort specified + if pagination is None or not pagination.sort: + messageDicts.sort(key=lambda x: x.get("publishedAt", getUtcTimestamp())) + + # Apply filtering (if filters provided) + if pagination and pagination.filters: + messageDicts = self._applyFilters(messageDicts, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination and pagination.sort: + messageDicts = self._applySorting(messageDicts, pagination.sort) + + # If no pagination requested, return all items + if pagination is None: + # Convert messages to ChatMessage objects and load documents + chat_messages = [] + for msg in messageDicts: + # Load documents from normalized documents table + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chat_message = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + chat_messages.append(chat_message) + + return chat_messages + + # Count total items after filters + totalItems = len(messageDicts) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedMessageDicts = messageDicts[startIdx:endIdx] + + # Convert messages to ChatMessage objects and load documents + chat_messages = [] + for msg in pagedMessageDicts: + # Load documents from normalized documents table + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chat_message = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + chat_messages.append(chat_message) + + return PaginatedResult( + items=chat_messages, + totalItems=totalItems, + totalPages=totalPages + ) + + def createMessage(self, messageData: Dict[str, Any]) -> ChatMessage: + """Creates a message for a workflow if user has access.""" + try: + # Ensure ID is present + if "id" not in messageData or not messageData["id"]: + messageData["id"] = f"msg_{uuid.uuid4()}" + # Check required fields + requiredFields = ["id", "workflowId"] + for field in requiredFields: + if field not in messageData: + logger.error(f"Required field '{field}' missing in messageData") + raise ValueError(f"Required field '{field}' missing in message data") + + # Check workflow access + workflowId = messageData["workflowId"] + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Validate that ID is not None + if messageData["id"] is None: + messageData["id"] = f"msg_{uuid.uuid4()}" + logger.warning(f"Automatically generated ID for workflow message: {messageData['id']}") + + # Set status if not present + if "status" not in messageData: + messageData["status"] = "step" # Default status for intermediate messages + + # Ensure role and agentName are present + if "role" not in messageData: + messageData["role"] = "assistant" if messageData.get("agentName") else "user" + + if "agentName" not in messageData: + messageData["agentName"] = "" + + # CRITICAL FIX: Automatically set roundNumber, taskNumber, and actionNumber if not provided + # This ensures messages have the correct progress context when workflows are continued + if "roundNumber" not in messageData: + messageData["roundNumber"] = workflow.currentRound + + if "taskNumber" not in messageData: + messageData["taskNumber"] = workflow.currentTask + + if "actionNumber" not in messageData: + messageData["actionNumber"] = workflow.currentAction + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in messageData or not messageData["mandateId"]: + messageData["mandateId"] = self.mandateId + if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: + messageData["featureInstanceId"] = self.featureInstanceId + + # Use generic field separation based on ChatMessage model + simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) + + # Handle documents separately - they will be stored in normalized documents table + documents_to_create = objectFields.get("documents", []) + + # Create message in normalized table using only simple fields + createdMessage = self.db.recordCreate(ChatMessage, simpleFields) + + + # Create documents in normalized documents table + created_documents = [] + logger.debug(f"Creating {len(documents_to_create)} document(s) for message {createdMessage['id']}") + for idx, doc_data in enumerate(documents_to_create): + try: + # Normalize to plain dict before assignment + if isinstance(doc_data, ChatDocument): + doc_dict = doc_data.model_dump() + elif isinstance(doc_data, dict): + doc_dict = dict(doc_data) + else: + # Attempt to coerce to ChatDocument then dump + try: + doc_dict = ChatDocument(**doc_data).model_dump() + except Exception as e: + logger.error(f"Invalid document data type for message creation (document {idx + 1}/{len(documents_to_create)}): {e}") + continue + + # Ensure messageId is set + doc_dict["messageId"] = createdMessage["id"] + logger.debug(f"Creating document {idx + 1}/{len(documents_to_create)}: fileName={doc_dict.get('fileName', 'unknown')}, fileId={doc_dict.get('fileId', 'unknown')}, messageId={doc_dict.get('messageId', 'unknown')}") + + created_doc = self.createDocument(doc_dict) + if created_doc: + created_documents.append(created_doc) + logger.debug(f"Successfully created document {idx + 1}/{len(documents_to_create)}: {created_doc.fileName} (id: {created_doc.id})") + else: + logger.error(f"Failed to create document {idx + 1}/{len(documents_to_create)}: createDocument returned None for fileName={doc_dict.get('fileName', 'unknown')}") + except Exception as e: + logger.error(f"Error processing document {idx + 1}/{len(documents_to_create)}: {e}", exc_info=True) + + logger.info(f"Created {len(created_documents)}/{len(documents_to_create)} document(s) for message {createdMessage['id']}") + + # Convert to ChatMessage model + chat_message = ChatMessage( + id=createdMessage["id"], + workflowId=createdMessage["workflowId"], + parentMessageId=createdMessage.get("parentMessageId"), + agentName=createdMessage.get("agentName"), + documents=created_documents, + documentsLabel=createdMessage.get("documentsLabel"), + message=createdMessage.get("message"), + role=createdMessage.get("role", "assistant"), + status=createdMessage.get("status", "step"), + sequenceNr=len(workflow.messages) + 1, # Use messages list length for sequence number + publishedAt=createdMessage.get("publishedAt", getUtcTimestamp()), + stats=objectFields.get("stats"), # Use stats from objectFields + roundNumber=createdMessage.get("roundNumber"), + taskNumber=createdMessage.get("taskNumber"), + actionNumber=createdMessage.get("actionNumber"), + success=createdMessage.get("success"), + actionId=createdMessage.get("actionId"), + actionMethod=createdMessage.get("actionMethod"), + actionName=createdMessage.get("actionName") + ) + + # Emit message event for streaming (if event manager is available) + try: + from modules.features.chatbot.eventManager import get_event_manager + event_manager = get_event_manager() + message_timestamp = parseTimestamp(chat_message.publishedAt, default=getUtcTimestamp()) + # Emit message event in exact chatData format: {type, createdAt, item} + asyncio.create_task(event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={ + "type": "message", + "createdAt": message_timestamp, + "item": chat_message.dict() + }, + event_category="chat" + )) + except Exception as e: + # Event manager not available or error - continue without emitting + logger.debug(f"Could not emit message event: {e}") + + # Debug: Store message and documents for debugging - only if debug enabled + storeDebugMessageAndDocuments(chat_message, self.currentUser) + + return chat_message + + except Exception as e: + logger.error(f"Error creating workflow message: {str(e)}") + return None + + def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatMessage]: + """Updates a workflow message if user has access to the workflow.""" + try: + + # Ensure messageId is provided + if not messageId: + logger.error("No messageId provided for updateMessage") + raise ValueError("messageId cannot be empty") + + # Check if message exists in database + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"id": messageId}) + if not messages: + logger.warning(f"Message with ID {messageId} does not exist in database") + + # If message doesn't exist but we have workflowId, create it + if "workflowId" in messageData: + workflowId = messageData.get("workflowId") + + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + logger.info(f"Creating new message with ID {messageId} for workflow {workflowId}") + return self.db.recordCreate(ChatMessage, messageData) + else: + logger.error(f"Workflow ID missing for new message {messageId}") + return None + + # Update existing message + existingMessage = messages[0] + + # Check workflow access + workflowId = existingMessage.get("workflowId") + workflow = self.getWorkflow(workflowId) + if not workflow: + raise PermissionError(f"No access to workflow {workflowId}") + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Use generic field separation based on ChatMessage model + simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) + + # Ensure required fields present + for key in ["role", "agentName"]: + if key not in simpleFields and key not in existingMessage: + simpleFields[key] = "assistant" if key == "role" else "" + + # Ensure ID is in the dataset + if 'id' not in simpleFields: + simpleFields['id'] = messageId + + # Convert createdAt to startedAt if needed + if "createdAt" in simpleFields and "startedAt" not in simpleFields: + simpleFields["startedAt"] = simpleFields["createdAt"] + del simpleFields["createdAt"] + + # Update the message with simple fields only + updatedMessage = self.db.recordModify(ChatMessage, messageId, simpleFields) + + # Handle object field updates (documents, stats) inline + if 'documents' in objectFields: + documents_data = objectFields['documents'] + try: + for doc_data in documents_data: + # Normalize to dict before mutation + if isinstance(doc_data, ChatDocument): + doc_dict = doc_data.model_dump() + elif isinstance(doc_data, dict): + doc_dict = dict(doc_data) + else: + try: + doc_dict = ChatDocument(**doc_data).model_dump() + except Exception: + logger.error("Invalid document data type for message update") + continue + doc_dict["messageId"] = messageId + self.createDocument(doc_dict) + except Exception as e: + logger.error(f"Error updating message documents: {str(e)}") + if not updatedMessage: + logger.warning(f"Failed to update message {messageId}") + return None + + # Convert to ChatMessage model + return ChatMessage(**updatedMessage) + except Exception as e: + logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) + raise ValueError(f"Error updating message {messageId}: {str(e)}") + + def deleteMessage(self, workflowId: str, messageId: str) -> bool: + """Deletes a workflow message and all related data if user has access to the workflow.""" + try: + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return False + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + # Check if the message exists + messages = self.getMessages(workflowId) + message = next((m for m in messages if m.get("id") == messageId), None) + + if not message: + logger.warning(f"Message {messageId} for workflow {workflowId} not found") + return False + + # CASCADE DELETE: Delete all related data first + + # 1. Delete message stats + existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) + for stat in existing_stats: + self.db.recordDelete(ChatStat, stat["id"]) + + # 2. Delete message documents (but NOT the files!) + existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + for doc in existing_docs: + self.db.recordDelete(ChatDocument, doc["id"]) + + # 3. Finally delete the message itself + success = self.db.recordDelete(ChatMessage, messageId) + + return success + + except Exception as e: + logger.error(f"Error deleting message {messageId}: {str(e)}") + return False + + def deleteFileFromMessage(self, workflowId: str, messageId: str, fileId: str) -> bool: + """Removes a file reference from a message if user has access.""" + try: + # Check workflow access + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return False + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + raise PermissionError(f"No permission to modify workflow {workflowId}") + + + # Get documents for this message from normalized table + documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + + if not documents: + logger.warning(f"No documents found for message {messageId}") + return False + + # Find and delete the specific document + removed = False + for doc in documents: + docId = doc.get("id") + fileIdValue = doc.get("fileId") + + # Flexible matching approach + shouldRemove = ( + (docId == fileId) or + (fileIdValue == fileId) or + (isinstance(docId, str) and str(fileId) in docId) or + (isinstance(fileIdValue, str) and str(fileId) in fileIdValue) + ) + + if shouldRemove: + # Delete the document from normalized table + success = self.db.recordDelete(ChatDocument, docId) + if success: + removed = True + else: + logger.warning(f"Failed to delete document {docId}") + + if not removed: + logger.warning(f"No matching file {fileId} found in message {messageId}") + return False + + except Exception as e: + logger.error(f"Error removing file {fileId} from message {messageId}: {str(e)}") + return False + + # Document methods + + def getDocuments(self, messageId: str) -> List[ChatDocument]: + """Returns documents for a message from normalized table.""" + try: + documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + return [ChatDocument(**doc) for doc in documents] + except Exception as e: + logger.error(f"Error getting message documents: {str(e)}") + return [] + + def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: + """Creates a document for a message in normalized table.""" + try: + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in documentData or not documentData["mandateId"]: + documentData["mandateId"] = self.mandateId + if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: + documentData["featureInstanceId"] = self.featureInstanceId + + # Validate and normalize document data to dict + document = ChatDocument(**documentData) + logger.debug(f"Creating document in database: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") + created = self.db.recordCreate(ChatDocument, document.model_dump()) + + if created: + created_doc = ChatDocument(**created) + logger.debug(f"Successfully created document in database: {created_doc.fileName} (id: {created_doc.id})") + return created_doc + else: + logger.error(f"Failed to create document in database: recordCreate returned None for fileName={document.fileName}") + return None + except Exception as e: + logger.error(f"Error creating message document: {str(e)}", exc_info=True) + return None + + + # Log methods + + def getLogs(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatLog], PaginatedResult]: + """ + Returns logs for a workflow if user has access to the workflow. + Supports optional pagination, sorting, and filtering. + + Args: + workflowId: The workflow ID to get logs for + pagination: Optional pagination parameters. If None, returns all items. + + Returns: + If pagination is None: List[ChatLog] + If pagination is provided: PaginatedResult with items and metadata + """ + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) + + # Get logs for this workflow from normalized table + logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + + # Convert raw logs to dict format for sorting/filtering + logDicts = [] + for log in logs: + logDicts.append({ + "id": log.get("id"), + "workflowId": log.get("workflowId"), + "message": log.get("message"), + "type": log.get("type"), + "timestamp": log.get("timestamp", getUtcTimestamp()), + "agentName": log.get("agentName"), + "status": log.get("status"), + "progress": log.get("progress"), + "mandateId": log.get("mandateId"), + "userId": log.get("userId") + }) + + # Apply default sorting by timestamp if no sort specified + if pagination is None or not pagination.sort: + logDicts.sort(key=lambda x: parseTimestamp(x.get("timestamp"), default=0)) + + # Apply filtering (if filters provided) + if pagination and pagination.filters: + logDicts = self._applyFilters(logDicts, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination and pagination.sort: + logDicts = self._applySorting(logDicts, pagination.sort) + + # If no pagination requested, return all items + if pagination is None: + return [ChatLog(**log) for log in logDicts] + + # Count total items after filters + totalItems = len(logDicts) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedLogDicts = logDicts[startIdx:endIdx] + + # Convert to model objects + items = [ChatLog(**log) for log in pagedLogDicts] + + return PaginatedResult( + items=items, + totalItems=totalItems, + totalPages=totalPages + ) + + def createLog(self, logData: Dict[str, Any]) -> ChatLog: + """Creates a log entry for a workflow if user has access.""" + # Check workflow access + workflowId = logData.get("workflowId") + if not workflowId: + logger.error("No workflowId provided for createLog") + return None + + workflow = self.getWorkflow(workflowId) + if not workflow: + logger.warning(f"No access to workflow {workflowId}") + return None + + if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): + logger.warning(f"No permission to modify workflow {workflowId}") + return None + + # Make sure required fields are present + if "timestamp" not in logData: + logData["timestamp"] = getUtcTimestamp() + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in logData or not logData["mandateId"]: + logData["mandateId"] = self.mandateId + if "featureInstanceId" not in logData or not logData["featureInstanceId"]: + logData["featureInstanceId"] = self.featureInstanceId + + # Add status information if not present + if "status" not in logData and "type" in logData: + if logData["type"] == "error": + logData["status"] = "error" + else: + logData["status"] = "running" + + # Add progress information if not present + if "progress" not in logData: + # Default progress values based on log type (0.0 to 1.0 format) + if logData.get("type") == "info": + logData["progress"] = 0.5 # Default middle progress + elif logData.get("type") == "error": + logData["progress"] = 1.0 # Error state - completed (failed) + elif logData.get("type") == "warning": + logData["progress"] = 0.5 # Default middle progress + + # Validate log data against ChatLog model + try: + log_model = ChatLog(**logData) + except Exception as e: + logger.error(f"Invalid log data: {str(e)}") + return None + + # Create log in normalized table + createdLog = self.db.recordCreate(ChatLog, log_model) + + # Emit log event for streaming (only for chatbot workflows) + # Only emit events for chatbot workflows, not for automation or dynamic workflows + if workflow.workflowMode == WorkflowModeEnum.WORKFLOW_CHATBOT: + try: + from modules.features.chatbot.eventManager import get_event_manager + event_manager = get_event_manager() + log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) + # Emit log event in exact chatData format: {type, createdAt, item} + asyncio.create_task(event_manager.emit_event( + workflowId, + "chatdata", + "New log", + "log", + { + "type": "log", + "createdAt": log_timestamp, + "item": ChatLog(**createdLog).model_dump() + } + )) + except Exception as e: + # Event manager not available or error - continue without emitting + logger.debug(f"Could not emit log event: {e}") + + # Return validated ChatLog instance + return ChatLog(**createdLog) + + # Stats methods + + def getStats(self, workflowId: str) -> List[ChatStat]: + """Returns list of statistics for a workflow if user has access.""" + # Check workflow access first (without calling getWorkflow to avoid circular reference) + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + return [] + + # Get stats for this workflow from normalized table + stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) + + if not stats: + return [] + + # Return all stats records sorted by creation time + stats.sort(key=lambda x: x.get("created_at", "")) + return [ChatStat(**stat) for stat in stats] + + + def createStat(self, statData: Dict[str, Any]) -> ChatStat: + """Creates a new stats record and returns it.""" + try: + # Ensure workflowId is present in statData + if "workflowId" not in statData: + raise ValueError("workflowId is required in statData") + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in statData or not statData["mandateId"]: + statData["mandateId"] = self.mandateId + if "featureInstanceId" not in statData or not statData["featureInstanceId"]: + statData["featureInstanceId"] = self.featureInstanceId + + # Validate the stat data against ChatStat model + stat = ChatStat(**statData) + + # Create the stat record in the database + created = self.db.recordCreate(ChatStat, stat) + + # Return the created ChatStat + return ChatStat(**created) + except Exception as e: + logger.error(f"Error creating workflow stat: {str(e)}") + raise + + + def getUnifiedChatData(self, workflowId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: + """ + Returns unified chat data (messages, logs, stats) for a workflow in chronological order. + Uses timestamp-based selective data transfer for efficient polling. + """ + # Check workflow access first + # Use RBAC filtering + workflows = getRecordsetWithRBAC(self.db, + ChatWorkflow, + self.currentUser, + recordFilter={"id": workflowId} + ) + + if not workflows: + return {"items": []} + + # Get all data types and filter in Python (PostgreSQL connector doesn't support $gt operators) + items = [] + + # Get messages + messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + for msg in messages: + # Apply timestamp filtering in Python + msgTimestamp = parseTimestamp(msg.get("publishedAt"), default=getUtcTimestamp()) + if afterTimestamp is not None and msgTimestamp <= afterTimestamp: + continue + + # Load documents for each message + documents = self.getDocuments(msg["id"]) + + # Create ChatMessage object with loaded documents + chatMessage = ChatMessage( + id=msg["id"], + workflowId=msg["workflowId"], + parentMessageId=msg.get("parentMessageId"), + documents=documents, + documentsLabel=msg.get("documentsLabel"), + message=msg.get("message"), + role=msg.get("role", "assistant"), + status=msg.get("status", "step"), + sequenceNr=msg.get("sequenceNr", 0), + publishedAt=msg.get("publishedAt", getUtcTimestamp()), + success=msg.get("success"), + actionId=msg.get("actionId"), + actionMethod=msg.get("actionMethod"), + actionName=msg.get("actionName"), + roundNumber=msg.get("roundNumber"), + taskNumber=msg.get("taskNumber"), + actionNumber=msg.get("actionNumber"), + taskProgress=msg.get("taskProgress"), + actionProgress=msg.get("actionProgress") + ) + + # Use publishedAt as the timestamp for chronological ordering + items.append({ + "type": "message", + "createdAt": msgTimestamp, + "item": chatMessage + }) + + # Get logs - return all logs with roundNumber if available + logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + for log in logs: + # Apply timestamp filtering in Python + logTimestamp = parseTimestamp(log.get("timestamp"), default=getUtcTimestamp()) + if afterTimestamp is not None and logTimestamp <= afterTimestamp: + continue + + chatLog = ChatLog(**log) + items.append({ + "type": "log", + "createdAt": logTimestamp, + "item": chatLog + }) + + # Get stats list + stats = self.getStats(workflowId) + for stat in stats: + # Apply timestamp filtering in Python + stat_timestamp = stat.createdAt if hasattr(stat, 'createdAt') else getUtcTimestamp() + if afterTimestamp is not None and stat_timestamp <= afterTimestamp: + continue + + items.append({ + "type": "stat", + "createdAt": stat_timestamp, + "item": stat + }) + + # Sort all items by createdAt timestamp for chronological order + items.sort(key=lambda x: parseTimestamp(x.get("createdAt"), default=0)) + + return {"items": items} + + # ===== Automation Methods ===== + + def _computeAutomationStatus(self, automation: Dict[str, Any]) -> str: + """Compute status field based on eventId presence""" + eventId = automation.get("eventId") + return "Running" if eventId else "Idle" + + def _enrichAutomationsWithUserAndMandate(self, automations: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Batch enrich automations with user names and mandate names for display. + Uses AppObjects interface to fetch users and mandates with proper access control. + """ + if not automations: + return automations + + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + + # Collect all unique user IDs and mandate IDs + userIds = set() + mandateIds = set() + + for automation in automations: + createdBy = automation.get("_createdBy") + if createdBy: + userIds.add(createdBy) + + mandateId = automation.get("mandateId") + if mandateId: + mandateIds.add(mandateId) + + # Use AppObjects interface to fetch users (respects access control) + appInterface = getAppInterface(self.currentUser) + usersMap = {} + if userIds: + for user_id in userIds: + user = appInterface.getUser(user_id) + if user: + usersMap[user_id] = user.username or user.email or user_id + + # Use AppObjects interface to fetch mandates (respects access control) + mandatesMap = {} + if mandateIds: + for mandate_id in mandateIds: + mandate = appInterface.getMandate(mandate_id) + if mandate: + mandatesMap[mandate_id] = mandate.name or mandate_id + + # Enrich each automation with the fetched data + for automation in automations: + createdBy = automation.get("_createdBy") + if createdBy: + automation["_createdByUserName"] = usersMap.get(createdBy, createdBy) + else: + automation["_createdByUserName"] = "-" + + mandateId = automation.get("mandateId") + if mandateId: + automation["mandateName"] = mandatesMap.get(mandateId, mandateId) + else: + automation["mandateName"] = "-" + + return automations + + def _enrichAutomationWithUserAndMandate(self, automation: Dict[str, Any]) -> Dict[str, Any]: + """ + Enrich a single automation with user name and mandate name for display. + For multiple automations, use _enrichAutomationsWithUserAndMandate for better performance. + """ + return self._enrichAutomationsWithUserAndMandate([automation])[0] + + def getAllAutomationDefinitions(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + """ + Returns automation definitions based on user access level. + Supports optional pagination, sorting, and filtering. + Computes status field for each automation. + """ + # Use RBAC filtering + filteredAutomations = getRecordsetWithRBAC(self.db, + AutomationDefinition, + self.currentUser + ) + + # Compute status for each automation and normalize executionLogs + for automation in filteredAutomations: + automation["status"] = self._computeAutomationStatus(automation) + # Ensure executionLogs is always a list, not None + if automation.get("executionLogs") is None: + automation["executionLogs"] = [] + + # Batch enrich with user and mandate names + self._enrichAutomationsWithUserAndMandate(filteredAutomations) + + # If no pagination requested, return all items + if pagination is None: + return filteredAutomations + + # Apply filtering (if filters provided) + if pagination.filters: + filteredAutomations = self._applyFilters(filteredAutomations, pagination.filters) + + # Apply sorting (in order of sortFields) + if pagination.sort: + filteredAutomations = self._applySorting(filteredAutomations, pagination.sort) + + # Count total items after filters + totalItems = len(filteredAutomations) + totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 + + # Apply pagination (skip/limit) + startIdx = (pagination.page - 1) * pagination.pageSize + endIdx = startIdx + pagination.pageSize + pagedAutomations = filteredAutomations[startIdx:endIdx] + + return PaginatedResult( + items=pagedAutomations, + totalItems=totalItems, + totalPages=totalPages + ) + + def getAutomationDefinition(self, automationId: str) -> Optional[AutomationDefinition]: + """Returns an automation definition by ID if user has access, with computed status.""" + try: + # Use RBAC filtering + filtered = getRecordsetWithRBAC(self.db, + AutomationDefinition, + self.currentUser, + recordFilter={"id": automationId} + ) + + if not filtered: + return None + + automation = filtered[0] + automation["status"] = self._computeAutomationStatus(automation) + # Ensure executionLogs is always a list, not None + if automation.get("executionLogs") is None: + automation["executionLogs"] = [] + # Enrich with user and mandate names + self._enrichAutomationWithUserAndMandate(automation) + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in automation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error getting automation definition: {str(e)}") + return None + + def createAutomationDefinition(self, automationData: Dict[str, Any]) -> AutomationDefinition: + """Creates a new automation definition, then triggers sync.""" + try: + # Ensure ID is present + if "id" not in automationData or not automationData["id"]: + automationData["id"] = str(uuid.uuid4()) + + # Ensure mandateId and featureInstanceId are set for proper data isolation + if "mandateId" not in automationData: + automationData["mandateId"] = self.mandateId + if "featureInstanceId" not in automationData: + automationData["featureInstanceId"] = self.featureInstanceId + + # Ensure database connector has correct userId context + # The connector should have been initialized with userId, but ensure it's updated + if self.userId and hasattr(self.db, 'updateContext'): + try: + self.db.updateContext(self.userId) + except Exception as e: + logger.warning(f"Could not update database context: {e}") + + # Note: _createdBy will be set automatically by connector's _saveRecord method + # when _createdAt is not present. We don't need to set it manually here. + # Use generic field separation + simpleFields, objectFields = self._separateObjectFields(AutomationDefinition, automationData) + + # Create automation in database + createdAutomation = self.db.recordCreate(AutomationDefinition, simpleFields) + + # Compute status + createdAutomation["status"] = self._computeAutomationStatus(createdAutomation) + # Ensure executionLogs is always a list, not None + if createdAutomation.get("executionLogs") is None: + createdAutomation["executionLogs"] = [] + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in createdAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error creating automation definition: {str(e)}") + raise + + def updateAutomationDefinition(self, automationId: str, automationData: Dict[str, Any]) -> AutomationDefinition: + """Updates an automation definition, then triggers sync.""" + try: + # Check access + existing = self.getAutomationDefinition(automationId) + if not existing: + raise PermissionError(f"No access to automation {automationId}") + + if not self.checkRbacPermission(AutomationDefinition, "update", automationId): + raise PermissionError(f"No permission to modify automation {automationId}") + + # Use generic field separation + simpleFields, objectFields = self._separateObjectFields(AutomationDefinition, automationData) + + # Update automation in database + updatedAutomation = self.db.recordModify(AutomationDefinition, automationId, simpleFields) + + # Compute status + updatedAutomation["status"] = self._computeAutomationStatus(updatedAutomation) + # Ensure executionLogs is always a list, not None + if updatedAutomation.get("executionLogs") is None: + updatedAutomation["executionLogs"] = [] + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + # Clean metadata fields and return Pydantic model + cleanedRecord = {k: v for k, v in updatedAutomation.items() if not k.startswith("_")} + return AutomationDefinition(**cleanedRecord) + except Exception as e: + logger.error(f"Error updating automation definition: {str(e)}") + raise + + def deleteAutomationDefinition(self, automationId: str) -> bool: + """Deletes an automation definition, then triggers sync.""" + try: + # Check access + existing = self.getAutomationDefinition(automationId) + if not existing: + raise PermissionError(f"No access to automation {automationId}") + + if not self.checkRbacPermission(AutomationDefinition, "delete", automationId): + raise PermissionError(f"No permission to delete automation {automationId}") + + # Remove event if exists + if existing.get("eventId"): + from modules.shared.eventManagement import eventManager + try: + eventManager.remove(existing["eventId"]) + except Exception as e: + logger.warning(f"Error removing event {existing['eventId']}: {str(e)}") + + # Delete automation from database + self.db.recordDelete(AutomationDefinition, automationId) + + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) + + return True + except Exception as e: + logger.error(f"Error deleting automation definition: {str(e)}") + raise + + def getAllAutomationDefinitionsWithRBAC(self, user: User) -> List[Dict[str, Any]]: + """ + Get all automation definitions filtered by RBAC for a specific user. + This method encapsulates getRecordsetWithRBAC() to avoid exposing the connector. + + Args: + user: User object for RBAC filtering + + Returns: + List of automation definition dictionaries filtered by RBAC + """ + return getRecordsetWithRBAC( + self.db, + AutomationDefinition, + user + ) + + async def _notifyAutomationChanged(self): + """Notify registered callbacks about automation changes (decoupled from features).""" + try: + from modules.shared.callbackRegistry import callbackRegistry + # Trigger callbacks without knowing which features are listening + await callbackRegistry.trigger('automation.changed', self) + except Exception as e: + logger.error(f"Error notifying automation change: {str(e)}") + + +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': + """ + Returns a ChatObjects instance for the current user. + Handles initialization of database and records. + + Args: + currentUser: The authenticated user + mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). + """ + if not currentUser: + raise ValueError("Invalid user context: user is required") + + effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None + + # Create context key including featureInstanceId for proper isolation + contextKey = f"{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" + + # Create new instance if not exists + if contextKey not in _chatInterfaces: + _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + else: + # Update user context if needed + _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + + return _chatInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbChatbot.py b/modules/interfaces/interfaceDbChatbot.py index c9f87a55..44f124b5 100644 --- a/modules/interfaces/interfaceDbChatbot.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -1,8 +1,8 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Interface to LucyDOM database and AI Connectors. -Uses the JSON connector for data access with added language support. +Interface to Chatbot database and AI Connectors. +Uses the PostgreSQL connector for data access with user/mandate filtering. """ import logging @@ -59,7 +59,7 @@ def storeDebugMessageAndDocuments(message, currentUser) -> None: import os from datetime import datetime, UTC from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir - from modules.interfaces.interfaceDbComponentObjects import getInterface + from modules.interfaces.interfaceDbManagement import getInterface # Create base debug directory (use base debug dir, not prompts subdirectory) baseDebugDir = _getBaseDebugDir() @@ -314,7 +314,7 @@ class ChatObjects: try: # Get configuration values with defaults dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") - dbDatabase = "poweron_chat" + dbDatabase = "poweron_chatbot" dbUser = APP_CONFIG.get("DB_USER") dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) @@ -1668,7 +1668,7 @@ class ChatObjects: if not automations: return automations - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface # Collect all unique user IDs and mandate IDs userIds = set() diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbManagement.py similarity index 99% rename from modules/interfaces/interfaceDbComponentObjects.py rename to modules/interfaces/interfaceDbManagement.py index 72d65839..b514cbf3 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -215,7 +215,7 @@ class ComponentObjects: # Get the root interface to access the initial mandate ID from modules.security.rootAccess import getRootUser - from modules.interfaces.interfaceDbAppObjects import getInterface + from modules.interfaces.interfaceDbApp import getInterface rootUser = getRootUser() rootInterface = getInterface(rootUser) diff --git a/modules/routes/routeAdmin.py b/modules/routes/routeAdmin.py index a60527e1..878dbd66 100644 --- a/modules/routes/routeAdmin.py +++ b/modules/routes/routeAdmin.py @@ -12,7 +12,7 @@ from fastapi import HTTPException, status from modules.shared.configuration import APP_CONFIG from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface # Static folder setup - using absolute path from app root baseDir = FilePath(__file__).parent.parent.parent # Go up to gateway root diff --git a/modules/routes/routeFeatureWorkflow.py b/modules/routes/routeAdminAutomationEvents.py similarity index 94% rename from modules/routes/routeFeatureWorkflow.py rename to modules/routes/routeAdminAutomationEvents.py index 33b9c0fc..c62977aa 100644 --- a/modules/routes/routeFeatureWorkflow.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User @@ -76,8 +76,8 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + from modules.interfaces.interfaceDbApp import getRootInterface from modules.features.workflow import syncAutomationEvents chatInterface = getChatInterface(currentUser) @@ -127,7 +127,7 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatbot.getInterface(currentUser) + chatInterface = interfaceDbChat.getInterface(currentUser) automation = chatInterface.getAutomationDefinition(automation_id) if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeAdminFeatures.py similarity index 99% rename from modules/routes/routeFeatures.py rename to modules/routes/routeAdminFeatures.py index 0659d919..d62a48e1 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelFeatures import Feature, FeatureInstance -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface logger = logging.getLogger(__name__) diff --git a/modules/routes/routeRbacExport.py b/modules/routes/routeAdminRbacExport.py similarity index 99% rename from modules/routes/routeRbacExport.py rename to modules/routes/routeAdminRbacExport.py index e7cc7204..11932f18 100644 --- a/modules/routes/routeRbacExport.py +++ b/modules/routes/routeAdminRbacExport.py @@ -21,7 +21,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelRbac import Role, AccessRule -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index b7069f86..c5c45963 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -17,7 +17,7 @@ from modules.auth import limiter, requireSysAdmin from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelRbac import Role from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeRbac.py b/modules/routes/routeAdminRbacRules.py similarity index 99% rename from modules/routes/routeRbac.py rename to modules/routes/routeAdminRbacRules.py index 7ab9b229..f16a9bc7 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeAdminRbacRules.py @@ -20,7 +20,7 @@ from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestCon from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 3a3e3ab4..db6affab 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -13,7 +13,7 @@ import logging import json # Import interfaces and models -from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface +from modules.interfaces.interfaceDbChat import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index ea660554..f39b6638 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -21,9 +21,9 @@ from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority, from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.interfaces.interfaceDbApp import getInterface from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp -from modules.interfaces.interfaceDbComponentObjects import ComponentObjects +from modules.interfaces.interfaceDbManagement import ComponentObjects # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 23db7170..66345ce5 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -10,7 +10,7 @@ import json from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelFiles import FileItem, FilePreview from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.datamodels.datamodelUam import User @@ -69,7 +69,7 @@ async def get_files( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllFiles(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -112,17 +112,17 @@ async def upload_file( file.fileName = file.filename """Upload a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Read file fileContent = await file.read() # Check size limits - maxSize = int(interfaceDbComponentObjects.APP_CONFIG.get("File_Management_MAX_UPLOAD_SIZE_MB")) * 1024 * 1024 # in bytes + maxSize = int(interfaceDbManagement.APP_CONFIG.get("File_Management_MAX_UPLOAD_SIZE_MB")) * 1024 * 1024 # in bytes if len(fileContent) > maxSize: raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=f"File too large. Maximum size: {interfaceDbComponentObjects.APP_CONFIG.get('File_Management_MAX_UPLOAD_SIZE_MB')}MB" + detail=f"File too large. Maximum size: {interfaceDbManagement.APP_CONFIG.get('File_Management_MAX_UPLOAD_SIZE_MB')}MB" ) # Save file via LucyDOM interface in the database @@ -155,7 +155,7 @@ async def upload_file( "isDuplicate": duplicateType != "new_file" }) - except interfaceDbComponentObjects.FileStorageError as e: + except interfaceDbManagement.FileStorageError as e: logger.error(f"Error during file upload (storage): {str(e)}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -177,7 +177,7 @@ async def get_file( ) -> FileItem: """Get a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file via LucyDOM interface from the database fileData = managementInterface.getFile(fileId) @@ -189,19 +189,19 @@ async def get_file( return fileData - except interfaceDbComponentObjects.FileNotFoundError as e: + except interfaceDbManagement.FileNotFoundError as e: logger.warning(f"File not found: {str(e)}") raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=str(e) ) - except interfaceDbComponentObjects.FilePermissionError as e: + except interfaceDbManagement.FilePermissionError as e: logger.warning(f"No permission for file: {str(e)}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=str(e) ) - except interfaceDbComponentObjects.FileError as e: + except interfaceDbManagement.FileError as e: logger.error(f"Error retrieving file: {str(e)}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -224,7 +224,7 @@ async def update_file( ) -> FileItem: """Update file info""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get the file from the database file = managementInterface.getFile(fileId) @@ -270,7 +270,7 @@ async def delete_file( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a file""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the file exists existingFile = managementInterface.getFile(fileId) @@ -297,7 +297,7 @@ async def get_file_stats( ) -> Dict[str, Any]: """Returns statistics about the stored files""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get all files - metadata only allFiles = managementInterface.getAllFiles() @@ -336,7 +336,7 @@ async def download_file( ) -> Response: """Download a file""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file data fileData = managementInterface.getFile(fileId) @@ -384,7 +384,7 @@ async def preview_file( ) -> FilePreview: """Preview a file's content""" try: - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get file preview using the correct method preview = managementInterface.getFileContent(fileId) diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 82ed3ad6..ab0ad8c1 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -20,7 +20,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, requireSysAdmin, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects +import modules.interfaces.interfaceDbApp as interfaceDbApp from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.shared.auditLogger import audit_logger @@ -107,7 +107,7 @@ async def get_mandates( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() result = appInterface.getAllMandates(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -150,7 +150,7 @@ async def get_mandate( MULTI-TENANT: SysAdmin-only. """ try: - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() mandate = appInterface.getMandate(mandateId) if not mandate: @@ -195,7 +195,7 @@ async def create_mandate( description = mandateData.get('description') enabled = mandateData.get('enabled', True) - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Create mandate newMandate = appInterface.createMandate( @@ -237,7 +237,7 @@ async def update_mandate( try: logger.debug(f"Updating mandate {mandateId} with data: {mandateData}") - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -280,7 +280,7 @@ async def delete_mandate( MULTI-TENANT: SysAdmin-only. """ try: - appInterface = interfaceDbAppObjects.getRootInterface() + appInterface = interfaceDbApp.getRootInterface() # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) @@ -347,7 +347,7 @@ async def listMandateUsers( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -518,7 +518,7 @@ async def addUserToMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # 3. Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -627,7 +627,7 @@ async def removeUserFromMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Verify mandate exists mandate = rootInterface.getMandate(targetMandateId) @@ -707,7 +707,7 @@ async def updateUserRolesInMandate( ) try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() # Get user's membership membership = rootInterface.getUserMandate(targetUserId, targetMandateId) @@ -810,7 +810,7 @@ def _hasMandateAdminRole(context: RequestContext, mandateId: str) -> bool: return False try: - rootInterface = interfaceDbAppObjects.getRootInterface() + rootInterface = interfaceDbApp.getRootInterface() for roleId in context.roleIds: roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index d3736b75..48902e66 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -10,7 +10,7 @@ import json from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelUtils import Prompt from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict @@ -58,7 +58,7 @@ async def get_prompts( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllPrompts(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -89,7 +89,7 @@ async def create_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Create a new prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Create prompt newPrompt = managementInterface.createPrompt(prompt) @@ -104,7 +104,7 @@ async def get_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Get a specific prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Get prompt prompt = managementInterface.getPrompt(promptId) @@ -125,7 +125,7 @@ async def update_prompt( currentUser: User = Depends(getCurrentUser) ) -> Prompt: """Update an existing prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the prompt exists existingPrompt = managementInterface.getPrompt(promptId) @@ -160,7 +160,7 @@ async def delete_prompt( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a prompt""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) # Check if the prompt exists existingPrompt = managementInterface.getPrompt(promptId) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 62a9b344..dab8bde9 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -17,7 +17,7 @@ import logging import json # Import interfaces and models -import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects +import modules.interfaces.interfaceDbApp as interfaceDbApp from modules.auth import limiter, getRequestContext, RequestContext # Import the attribute definition and helper functions @@ -179,7 +179,7 @@ async def get_users( detail=f"Invalid pagination parameter: {str(e)}" ) - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # MULTI-TENANT: Use mandateId from context (header) # SysAdmin without mandateId can see all users @@ -278,7 +278,7 @@ async def get_user( MULTI-TENANT: User must be in the same mandate (via UserMandate) or caller is SysAdmin. """ try: - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get user without filtering by enabled status user = appInterface.getUser(userId) @@ -333,7 +333,7 @@ async def create_user( Create a new user. MULTI-TENANT: User is created and automatically added to the current mandate. """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Extract fields from request model and call createUser with individual parameters from modules.datamodels.datamodelUam import AuthAuthority @@ -375,7 +375,7 @@ async def update_user( Update an existing user. MULTI-TENANT: Can only update users in the same mandate (unless SysAdmin). """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) @@ -430,7 +430,7 @@ async def reset_user_password( ) # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get target user target_user = appInterface.getUser(userId) @@ -525,7 +525,7 @@ async def change_password( """ try: # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Verify current password if not appInterface.verifyPassword(currentPassword, context.user.passwordHash): @@ -612,10 +612,10 @@ async def sendPasswordLink( """ try: from modules.shared.configuration import APP_CONFIG - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface # Get user interface - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Get target user targetUser = appInterface.getUser(userId) @@ -742,7 +742,7 @@ async def delete_user( Delete a user. MULTI-TENANT: Can only delete users in the same mandate (unless SysAdmin). """ - appInterface = interfaceDbAppObjects.getInterface(context.user) + appInterface = interfaceDbApp.getInterface(context.user) # Check if the user exists existingUser = appInterface.getUser(userId) diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeDataWorkflows.py similarity index 99% rename from modules/routes/routeWorkflows.py rename to modules/routes/routeDataWorkflows.py index ab9e1ff6..d3b2d825 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeDataWorkflows.py @@ -14,8 +14,8 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot -from modules.interfaces.interfaceDbChatbot import getInterface +import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.interfaces.interfaceDbChat import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models @@ -45,7 +45,7 @@ router = APIRouter( ) def getServiceChat(currentUser: User): - return interfaceDbChatbot.getInterface(currentUser) + return interfaceDbChat.getInterface(currentUser) # Consolidated endpoint for getting all workflows @router.get("/", response_model=PaginatedResponse[ChatWorkflow]) diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index ed1fd9f3..5be544a8 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum @@ -32,7 +32,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChat.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Workflow start endpoint @router.post("/start", response_model=ChatWorkflow) diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index b5b80e2e..977158f0 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -18,7 +18,7 @@ from modules.shared.timeUtils import parseTimestamp from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models @@ -43,7 +43,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChat.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Chatbot streaming endpoint (SSE) @router.post("/start/stream") diff --git a/modules/routes/routeFeatureRealEstate.py b/modules/routes/routeFeatureRealEstate.py index 7e130c1b..73364345 100644 --- a/modules/routes/routeFeatureRealEstate.py +++ b/modules/routes/routeFeatureRealEstate.py @@ -26,7 +26,7 @@ from modules.datamodels.datamodelRealEstate import ( ) # Import interfaces -from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands from modules.features.realEstate.mainRealEstate import ( diff --git a/modules/routes/routeFeatureTrustee.py b/modules/routes/routeFeatureTrustee.py index ad842db8..14b0a9e8 100644 --- a/modules/routes/routeFeatureTrustee.py +++ b/modules/routes/routeFeatureTrustee.py @@ -19,7 +19,7 @@ import io from modules.auth import limiter, getRequestContext, RequestContext from modules.interfaces.interfaceDbTrustee import getInterface -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface from modules.datamodels.datamodelTrustee import ( TrusteeOrganisation, diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index 53375849..f99dcd77 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -21,7 +21,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp from modules.shared.auditLogger import audit_logger diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index f649d2b2..3b059f9c 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, getCurrentUser from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelInvitation import Invitation -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index dae6e04e..953dd5f2 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -11,7 +11,7 @@ from modules.auth import limiter, getCurrentUser, getRequestContext, RequestCont from modules.datamodels.datamodelRbac import Role # Import interfaces -import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects +import modules.interfaces.interfaceDbManagement as interfaceDbManagement from modules.datamodels.datamodelMessaging import ( MessagingSubscription, MessagingSubscriptionRegistration, @@ -55,7 +55,7 @@ async def getSubscriptions( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllSubscriptions(pagination=paginationParams) if paginationParams: @@ -85,7 +85,7 @@ async def createSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Create a new subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) subscriptionData = subscription.model_dump(exclude={"id"}) newSubscription = managementInterface.createSubscription(subscriptionData) @@ -101,7 +101,7 @@ async def getSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Get a specific subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) subscription = managementInterface.getSubscription(subscriptionId) if not subscription: @@ -122,7 +122,7 @@ async def updateSubscription( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscription: """Update an existing subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingSubscription = managementInterface.getSubscription(subscriptionId) if not existingSubscription: @@ -151,7 +151,7 @@ async def deleteSubscription( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a subscription""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingSubscription = managementInterface.getSubscription(subscriptionId) if not existingSubscription: @@ -192,7 +192,7 @@ async def getSubscriptionRegistrations( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllRegistrations( subscriptionId=subscriptionId, pagination=paginationParams @@ -227,7 +227,7 @@ async def subscribeUser( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscriptionRegistration: """Subscribe user to a subscription with a specific channel""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) registration = managementInterface.subscribeUser( subscriptionId=subscriptionId, @@ -248,7 +248,7 @@ async def unsubscribeUser( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Unsubscribe user from a subscription for a specific channel""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) success = managementInterface.unsubscribeUser( subscriptionId=subscriptionId, @@ -284,7 +284,7 @@ async def getMyRegistrations( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getAllRegistrations( userId=currentUser.id, pagination=paginationParams @@ -318,7 +318,7 @@ async def updateRegistration( currentUser: User = Depends(getCurrentUser) ) -> MessagingSubscriptionRegistration: """Update a registration""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingRegistration = managementInterface.getRegistration(registrationId) if not existingRegistration: @@ -347,7 +347,7 @@ async def deleteRegistration( currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: """Delete a registration""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) existingRegistration = managementInterface.getRegistration(registrationId) if not existingRegistration: @@ -417,7 +417,7 @@ def _hasTriggerPermission(context: RequestContext) -> bool: return False try: - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() for roleId in context.roleIds: @@ -458,7 +458,7 @@ async def getDeliveries( detail=f"Invalid pagination parameter: {str(e)}" ) - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) result = managementInterface.getDeliveries( subscriptionId=subscriptionId, userId=currentUser.id, # Users can only see their own deliveries @@ -492,7 +492,7 @@ async def getDelivery( currentUser: User = Depends(getCurrentUser) ) -> MessagingDelivery: """Get a specific delivery""" - managementInterface = interfaceDbComponentObjects.getInterface(currentUser) + managementInterface = interfaceDbManagement.getInterface(currentUser) delivery = managementInterface.getDelivery(deliveryId) if not delivery: diff --git a/modules/routes/routeSecurityAdmin.py b/modules/routes/routeSecurityAdmin.py index 3ba35e65..36388c9f 100644 --- a/modules/routes/routeSecurityAdmin.py +++ b/modules/routes/routeSecurityAdmin.py @@ -13,7 +13,7 @@ import logging from modules.auth import getCurrentUser, limiter, requireSysAdmin from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 2a8f65fd..9642df00 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -13,7 +13,7 @@ from requests_oauthlib import OAuth2Session import httpx from modules.shared.configuration import APP_CONFIG -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection from modules.auth import getCurrentUser, limiter from modules.auth import createAccessToken, setAccessTokenCookie, createRefreshToken, setRefreshTokenCookie diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 96f22136..64984eef 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -16,7 +16,7 @@ from jose import jwt # Import auth modules from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 77dc9885..6d034607 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -13,7 +13,7 @@ import msal import httpx from modules.shared.configuration import APP_CONFIG -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter diff --git a/modules/routes/routeSharepoint.py b/modules/routes/routeSharepoint.py index 7915b0f1..aa62afc6 100644 --- a/modules/routes/routeSharepoint.py +++ b/modules/routes/routeSharepoint.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Query, Request, sta from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User, UserConnection -from modules.interfaces.interfaceDbAppObjects import getInterface +from modules.interfaces.interfaceDbApp import getInterface from modules.services import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index fb4e6512..ff91dc6b 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -49,13 +49,13 @@ class Services: # Initialize interfaces with explicit mandateId - from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbTrustee import getInterface as getTrusteeInterface diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 13739dea..199201eb 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -65,7 +65,7 @@ class ExtractionService: results: List[ContentExtracted] = [] # Lazy import to avoid circular deps and heavy init at module import - from modules.interfaces.interfaceDbComponentObjects import getInterface + from modules.interfaces.interfaceDbManagement import getInterface dbInterface = getInterface() totalDocs = len(documents) diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 5d3a8497..2c975f1d 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -157,11 +157,11 @@ class UtilsService: def storeDebugMessageAndDocuments(self, message, currentUser): """ - Wrapper to store debug messages and documents via interfaceDbChatbot. - Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatbot. + Wrapper to store debug messages and documents via interfaceDbChat. + Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChat. """ try: - from modules.interfaces.interfaceDbChatbot import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index a6e3a78a..136dd2cb 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -452,10 +452,10 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - from modules.interfaces.interfaceDbAppObjects import getRootInterface + import modules.interfaces.interfaceDbChat as interfaceDbChat + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() - interfaceDbChat = interfaceDbChatbot.getInterface(rootInterface.currentUser) + interfaceDbChat = interfaceDbChat.getInterface(rootInterface.currentUser) # Get workflow logs chatData = interfaceDbChat.getUnifiedChatData(context.workflow.id, None) diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index 05a3f34b..f807fe24 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -27,7 +27,7 @@ class MethodAiOperationsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -94,8 +94,8 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) # Import and initialize services from modules.services import getInterface as getServices @@ -201,8 +201,8 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,8 +283,8 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + import modules.interfaces.interfaceDbChat as interfaceDbChat + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 9da22d66..3e46bc0c 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -27,7 +27,7 @@ from modules.datamodels.datamodelWorkflow import AiResponse class AIBehaviorTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -45,7 +45,7 @@ class AIBehaviorTester: from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + import modules.interfaces.interfaceDbChat as interfaceDbChat currentTimestamp = time.time() @@ -67,7 +67,7 @@ class AIBehaviorTester: ) # SAVE workflow to database so it exists for access control - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 7beaeec4..8850ae2b 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -23,13 +23,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class WorkflowWithDocumentsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -192,7 +192,7 @@ class WorkflowWithDocumentsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) if not currentWorkflow: @@ -225,7 +225,7 @@ class WorkflowWithDocumentsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 698c9698..106e2999 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -25,13 +25,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class WorkflowPromptVariationsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -115,7 +115,7 @@ class WorkflowPromptVariationsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(workflow.id) if not currentWorkflow: @@ -140,7 +140,7 @@ class WorkflowPromptVariationsTester: def _analyzeWorkflowResults(self, workflow: Any) -> Dict[str, Any]: """Analyze workflow results and extract information.""" - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(workflow.id) if not workflow: diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index d9c4d9b8..3c85460e 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -24,13 +24,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class DocumentGenerationFormatsTester: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -251,7 +251,7 @@ class DocumentGenerationFormatsTester: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -296,7 +296,7 @@ class DocumentGenerationFormatsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 45a364ce..19bdb12f 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -24,13 +24,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class DocumentGenerationFormatsTester10: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -248,7 +248,7 @@ class DocumentGenerationFormatsTester10: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -293,7 +293,7 @@ class DocumentGenerationFormatsTester10: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 6d1735ad..64c8f93e 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -26,13 +26,13 @@ from modules.services import getInterface as getServices from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +import modules.interfaces.interfaceDbChat as interfaceDbChat class CodeGenerationFormatsTester11: def __init__(self): # Use root user for testing (has full access to everything) - from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() self.testUser = rootInterface.currentUser @@ -190,7 +190,7 @@ class CodeGenerationFormatsTester11: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -235,7 +235,7 @@ class CodeGenerationFormatsTester11: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) + interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/integration/options/test_options_api.py b/tests/integration/options/test_options_api.py index 68a276e2..7007bf2a 100644 --- a/tests/integration/options/test_options_api.py +++ b/tests/integration/options/test_options_api.py @@ -9,7 +9,7 @@ import pytest import secrets from fastapi.testclient import TestClient from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface @pytest.fixture From 362080791aa386e8254b9e9f7873c390503141f2 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 22 Jan 2026 17:00:29 +0100 Subject: [PATCH 12/32] isolate features --- app.py | 68 +- modules/datamodels/datamodelWorkflow.py | 6 - .../aichat}/aicore/aicoreBase.py | 0 .../aichat}/aicore/aicoreModelRegistry.py | 2 +- .../aichat}/aicore/aicoreModelSelector.py | 0 .../aichat}/aicore/aicorePluginAnthropic.py | 2 +- .../aichat}/aicore/aicorePluginInternal.py | 2 +- .../aichat}/aicore/aicorePluginOpenai.py | 2 +- .../aichat}/aicore/aicorePluginPerplexity.py | 2 +- .../aichat}/aicore/aicorePluginTavily.py | 2 +- .../aichat/datamodelFeatureAiChat.py} | 0 .../aichat/interfaceFeatureAiChat.py} | 53 +- modules/features/aichat/mainAiChat.py | 166 + .../aichat/routeFeatureAiChat.py} | 6 +- .../aichat}/serviceAi/mainServiceAi.py | 26 +- .../aichat}/serviceAi/merge_1.txt | 0 .../serviceAi/subAiCallLooping-flow.md | 0 .../aichat}/serviceAi/subAiCallLooping.py | 4 +- .../aichat}/serviceAi/subContentExtraction.py | 2 +- .../aichat}/serviceAi/subDocumentIntents.py | 2 +- .../aichat}/serviceAi/subJsonMerger.py | 0 .../serviceAi/subJsonResponseHandling.py | 2 +- .../aichat}/serviceAi/subLoopingUseCases.py | 0 .../aichat}/serviceAi/subResponseParsing.py | 2 +- .../aichat}/serviceAi/subStructureFilling.py | 2 +- .../serviceAi/subStructureGeneration.py | 2 +- .../aichat}/serviceExtraction/__init__.py | 0 .../serviceExtraction/chunking/__init__.py | 0 .../chunking/chunkerImage.py | 0 .../chunking/chunkerStructure.py | 0 .../chunking/chunkerTable.py | 0 .../serviceExtraction/chunking/chunkerText.py | 0 .../serviceExtraction/extractors/__init__.py | 0 .../extractors/extractorBinary.py | 0 .../extractors/extractorCsv.py | 0 .../extractors/extractorDocx.py | 0 .../extractors/extractorHtml.py | 0 .../extractors/extractorImage.py | 0 .../extractors/extractorJson.py | 0 .../extractors/extractorPdf.py | 0 .../extractors/extractorPptx.py | 0 .../extractors/extractorSql.py | 0 .../extractors/extractorText.py | 0 .../extractors/extractorXlsx.py | 0 .../extractors/extractorXml.py | 0 .../mainServiceExtraction.py | 6 +- .../serviceExtraction/merging/__init__.py | 0 .../merging/mergerDefault.py | 0 .../serviceExtraction/merging/mergerTable.py | 0 .../serviceExtraction/merging/mergerText.py | 0 .../aichat}/serviceExtraction/subMerger.py | 0 .../aichat}/serviceExtraction/subPipeline.py | 0 .../subPromptBuilderExtraction.py | 2 +- .../aichat}/serviceExtraction/subRegistry.py | 2 +- .../aichat}/serviceExtraction/subUtils.py | 0 .../mainServiceGeneration.py | 12 +- .../serviceGeneration/paths/codePath.py | 2 +- .../serviceGeneration/paths/documentPath.py | 0 .../serviceGeneration/paths/imagePath.py | 0 .../renderers/codeRendererBaseTemplate.py | 0 .../renderers/documentRendererBaseTemplate.py | 0 .../serviceGeneration/renderers/registry.py | 0 .../renderers/rendererCodeCsv.py | 0 .../renderers/rendererCodeJson.py | 0 .../renderers/rendererCodeXml.py | 0 .../renderers/rendererCsv.py | 0 .../renderers/rendererDocx.py | 0 .../renderers/rendererHtml.py | 0 .../renderers/rendererImage.py | 0 .../renderers/rendererJson.py | 0 .../renderers/rendererMarkdown.py | 0 .../renderers/rendererPdf.py | 0 .../renderers/rendererPptx.py | 0 .../renderers/rendererText.py | 0 .../renderers/rendererXlsx.py | 0 .../serviceGeneration/subContentGenerator.py | 2 +- .../serviceGeneration/subContentIntegrator.py | 0 .../serviceGeneration/subDocumentUtility.py | 0 .../serviceGeneration/subJsonSchema.py | 0 .../subPromptBuilderGeneration.py | 0 .../subStructureGenerator.py | 0 .../aichat}/serviceWeb/mainServiceWeb.py | 0 modules/features/automation/mainAutomation.py | 148 + .../automation/routeFeatureAutomation.py} | 8 +- .../subAutomationTemplates.py | 0 .../subAutomationUtils.py | 0 .../chatbot/datamodelFeatureChatbot.py} | 0 .../chatbot/interfaceFeatureChatbot.py} | 2 +- modules/features/chatbot/mainChatbot.py | 108 +- .../chatbot}/routeFeatureChatbot.py | 10 +- .../dynamicOptions/mainDynamicOptions.py | 237 -- modules/features/featureRegistry.py | 117 + modules/features/featuresLifecycle.py | 62 - .../datamodelFeatureNeutralizer.py} | 0 .../mainNeutralizePlayground.py | 2 +- .../features/neutralizer/mainNeutralizer.py | 125 + .../neutralizer/routeFeatureNeutralizer.py} | 4 +- .../mainServiceNeutralization.py | 12 +- .../serviceNeutralization/subParseString.py | 2 +- .../serviceNeutralization/subPatterns.py | 0 .../serviceNeutralization/subProcessBinary.py | 0 .../serviceNeutralization/subProcessCommon.py | 0 .../serviceNeutralization/subProcessList.py | 8 +- .../serviceNeutralization/subProcessText.py | 2 +- .../realEstate/datamodelFeatureRealEstate.py} | 0 .../realEstate/interfaceFeatureRealEstate.py} | 2 +- modules/features/realEstate/mainRealEstate.py | 150 +- .../realEstate}/routeFeatureRealEstate.py | 6 +- .../trustee/datamodelFeatureTrustee.py} | 0 .../trustee/interfaceFeatureTrustee.py} | 2 +- modules/features/trustee/mainTrustee.py | 193 ++ .../trustee}/routeFeatureTrustee.py | 4 +- modules/interfaces/interfaceAiObjects.py | 4 +- modules/interfaces/interfaceBootstrap.py | 419 --- modules/interfaces/interfaceDbApp.py | 2 +- modules/routes/routeAdminAutomationEvents.py | 8 +- modules/routes/routeDataWorkflows.py | 10 +- modules/routes/routeOptions.py | 88 - modules/security/rbacCatalog.py | 151 + modules/services/__init__.py | 165 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 2 +- .../automation}/__init__.py | 0 .../automation}/mainWorkflow.py | 2 +- .../automation/subAutomationSchedule.py | 64 + .../automation/subAutomationTemplates.py | 385 +++ .../automation/subAutomationUtils.py | 118 + .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 6 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 8 +- modules/workflows/workflowManager.py | 10 +- scripts/import_analysis.csv | 2716 +++++++++++++++++ scripts/script_analyze_imports.py | 196 ++ tests/functional/test01_ai_model_selection.py | 6 +- tests/functional/test02_ai_models.py | 16 +- tests/functional/test03_ai_operations.py | 12 +- tests/functional/test04_ai_behavior.py | 4 +- .../test05_workflow_with_documents.py | 6 +- .../test06_workflow_prompt_variations.py | 6 +- tests/functional/test07_json_merge.py | 2 +- tests/functional/test08_json_finalization.py | 2 +- .../test09_document_generation_formats.py | 6 +- .../test10_document_generation_formats.py | 6 +- .../test11_code_generation_formats.py | 6 +- tests/functional/test12_json_split_merge.py | 2 +- tests/functional/test_kpi_full.py | 2 +- tests/functional/test_kpi_incomplete.py | 2 +- tests/functional/test_kpi_path.py | 2 +- .../workflows/test_workflow_execution.py | 2 +- .../options/test_frontend_options_types.py | 117 - tests/unit/options/test_main_options.py | 183 -- .../services/test_json_extraction_merging.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- 195 files changed, 4966 insertions(+), 1461 deletions(-) rename modules/{ => features/aichat}/aicore/aicoreBase.py (100%) rename modules/{ => features/aichat}/aicore/aicoreModelRegistry.py (99%) rename modules/{ => features/aichat}/aicore/aicoreModelSelector.py (100%) rename modules/{ => features/aichat}/aicore/aicorePluginAnthropic.py (99%) rename modules/{ => features/aichat}/aicore/aicorePluginInternal.py (98%) rename modules/{ => features/aichat}/aicore/aicorePluginOpenai.py (99%) rename modules/{ => features/aichat}/aicore/aicorePluginPerplexity.py (99%) rename modules/{ => features/aichat}/aicore/aicorePluginTavily.py (99%) rename modules/{datamodels/datamodelChat.py => features/aichat/datamodelFeatureAiChat.py} (100%) rename modules/{interfaces/interfaceDbChat.py => features/aichat/interfaceFeatureAiChat.py} (97%) create mode 100644 modules/features/aichat/mainAiChat.py rename modules/{routes/routeFeatureChatDynamic.py => features/aichat/routeFeatureAiChat.py} (95%) rename modules/{services => features/aichat}/serviceAi/mainServiceAi.py (97%) rename modules/{services => features/aichat}/serviceAi/merge_1.txt (100%) rename modules/{services => features/aichat}/serviceAi/subAiCallLooping-flow.md (100%) rename modules/{services => features/aichat}/serviceAi/subAiCallLooping.py (99%) rename modules/{services => features/aichat}/serviceAi/subContentExtraction.py (99%) rename modules/{services => features/aichat}/serviceAi/subDocumentIntents.py (99%) rename modules/{services => features/aichat}/serviceAi/subJsonMerger.py (100%) rename modules/{services => features/aichat}/serviceAi/subJsonResponseHandling.py (99%) rename modules/{services => features/aichat}/serviceAi/subLoopingUseCases.py (100%) rename modules/{services => features/aichat}/serviceAi/subResponseParsing.py (99%) rename modules/{services => features/aichat}/serviceAi/subStructureFilling.py (99%) rename modules/{services => features/aichat}/serviceAi/subStructureGeneration.py (99%) rename modules/{services => features/aichat}/serviceExtraction/__init__.py (100%) rename modules/{services => features/aichat}/serviceExtraction/chunking/__init__.py (100%) rename modules/{services => features/aichat}/serviceExtraction/chunking/chunkerImage.py (100%) rename modules/{services => features/aichat}/serviceExtraction/chunking/chunkerStructure.py (100%) rename modules/{services => features/aichat}/serviceExtraction/chunking/chunkerTable.py (100%) rename modules/{services => features/aichat}/serviceExtraction/chunking/chunkerText.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/__init__.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorBinary.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorCsv.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorDocx.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorHtml.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorImage.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorJson.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorPdf.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorPptx.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorSql.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorText.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorXlsx.py (100%) rename modules/{services => features/aichat}/serviceExtraction/extractors/extractorXml.py (100%) rename modules/{services => features/aichat}/serviceExtraction/mainServiceExtraction.py (99%) rename modules/{services => features/aichat}/serviceExtraction/merging/__init__.py (100%) rename modules/{services => features/aichat}/serviceExtraction/merging/mergerDefault.py (100%) rename modules/{services => features/aichat}/serviceExtraction/merging/mergerTable.py (100%) rename modules/{services => features/aichat}/serviceExtraction/merging/mergerText.py (100%) rename modules/{services => features/aichat}/serviceExtraction/subMerger.py (100%) rename modules/{services => features/aichat}/serviceExtraction/subPipeline.py (100%) rename modules/{services => features/aichat}/serviceExtraction/subPromptBuilderExtraction.py (98%) rename modules/{services => features/aichat}/serviceExtraction/subRegistry.py (99%) rename modules/{services => features/aichat}/serviceExtraction/subUtils.py (100%) rename modules/{services => features/aichat}/serviceGeneration/mainServiceGeneration.py (98%) rename modules/{services => features/aichat}/serviceGeneration/paths/codePath.py (99%) rename modules/{services => features/aichat}/serviceGeneration/paths/documentPath.py (100%) rename modules/{services => features/aichat}/serviceGeneration/paths/imagePath.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/codeRendererBaseTemplate.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/documentRendererBaseTemplate.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/registry.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererCodeCsv.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererCodeJson.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererCodeXml.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererCsv.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererDocx.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererHtml.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererImage.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererJson.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererMarkdown.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererPdf.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererPptx.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererText.py (100%) rename modules/{services => features/aichat}/serviceGeneration/renderers/rendererXlsx.py (100%) rename modules/{services => features/aichat}/serviceGeneration/subContentGenerator.py (99%) rename modules/{services => features/aichat}/serviceGeneration/subContentIntegrator.py (100%) rename modules/{services => features/aichat}/serviceGeneration/subDocumentUtility.py (100%) rename modules/{services => features/aichat}/serviceGeneration/subJsonSchema.py (100%) rename modules/{services => features/aichat}/serviceGeneration/subPromptBuilderGeneration.py (100%) rename modules/{services => features/aichat}/serviceGeneration/subStructureGenerator.py (100%) rename modules/{services => features/aichat}/serviceWeb/mainServiceWeb.py (100%) create mode 100644 modules/features/automation/mainAutomation.py rename modules/{routes/routeDataAutomation.py => features/automation/routeFeatureAutomation.py} (96%) rename modules/features/{workflow => automation}/subAutomationTemplates.py (100%) rename modules/features/{workflow => automation}/subAutomationUtils.py (100%) rename modules/{datamodels/datamodelChatbot.py => features/chatbot/datamodelFeatureChatbot.py} (100%) rename modules/{interfaces/interfaceDbChatbot.py => features/chatbot/interfaceFeatureChatbot.py} (99%) rename modules/{routes => features/chatbot}/routeFeatureChatbot.py (98%) delete mode 100644 modules/features/dynamicOptions/mainDynamicOptions.py create mode 100644 modules/features/featureRegistry.py delete mode 100644 modules/features/featuresLifecycle.py rename modules/{datamodels/datamodelNeutralizer.py => features/neutralizer/datamodelFeatureNeutralizer.py} (100%) rename modules/features/{neutralizePlayground => neutralizer}/mainNeutralizePlayground.py (99%) create mode 100644 modules/features/neutralizer/mainNeutralizer.py rename modules/{routes/routeFeatureNeutralization.py => features/neutralizer/routeFeatureNeutralizer.py} (97%) rename modules/{services => features/neutralizer}/serviceNeutralization/mainServiceNeutralization.py (95%) rename modules/{services => features/neutralizer}/serviceNeutralization/subParseString.py (98%) rename modules/{services => features/neutralizer}/serviceNeutralization/subPatterns.py (100%) rename modules/{services => features/neutralizer}/serviceNeutralization/subProcessBinary.py (100%) rename modules/{services => features/neutralizer}/serviceNeutralization/subProcessCommon.py (100%) rename modules/{services => features/neutralizer}/serviceNeutralization/subProcessList.py (96%) rename modules/{services => features/neutralizer}/serviceNeutralization/subProcessText.py (97%) rename modules/{datamodels/datamodelRealEstate.py => features/realEstate/datamodelFeatureRealEstate.py} (100%) rename modules/{interfaces/interfaceDbRealEstate.py => features/realEstate/interfaceFeatureRealEstate.py} (99%) rename modules/{routes => features/realEstate}/routeFeatureRealEstate.py (99%) rename modules/{datamodels/datamodelTrustee.py => features/trustee/datamodelFeatureTrustee.py} (100%) rename modules/{interfaces/interfaceDbTrustee.py => features/trustee/interfaceFeatureTrustee.py} (99%) create mode 100644 modules/features/trustee/mainTrustee.py rename modules/{routes => features/trustee}/routeFeatureTrustee.py (99%) delete mode 100644 modules/routes/routeOptions.py create mode 100644 modules/security/rbacCatalog.py rename modules/{features/workflow => workflows/automation}/__init__.py (100%) rename modules/{features/workflow => workflows/automation}/mainWorkflow.py (99%) create mode 100644 modules/workflows/automation/subAutomationSchedule.py create mode 100644 modules/workflows/automation/subAutomationTemplates.py create mode 100644 modules/workflows/automation/subAutomationUtils.py create mode 100644 scripts/import_analysis.csv create mode 100644 scripts/script_analyze_imports.py delete mode 100644 tests/unit/options/test_frontend_options_types.py delete mode 100644 tests/unit/options/test_main_options.py diff --git a/app.py b/app.py index 6944b144..6581065b 100644 --- a/app.py +++ b/app.py @@ -19,8 +19,9 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager -from modules.features import featuresLifecycle as featuresLifecycle +from modules.workflows.automation import subAutomationSchedule from modules.interfaces.interfaceDbApp import getRootInterface +from modules.features.featureRegistry import loadFeatureMainModules class DailyRotatingFileHandler(RotatingFileHandler): """ @@ -282,18 +283,27 @@ instanceLabel = APP_CONFIG.get("APP_ENV_LABEL") async def lifespan(app: FastAPI): logger.info("Application is starting up") - # Initialize AI connectors once at startup to avoid per-request discovery - from modules.aicore.aicoreModelRegistry import modelRegistry - modelRegistry.ensureConnectorsRegistered() - # Get event user for feature lifecycle (system-level user for background operations) rootInterface = getRootInterface() eventUser = rootInterface.getUserByUsername("event") if not eventUser: logger.error("Could not get event user - some features may not start properly") + # --- Init Feature Containers (Plug&Play) --- + try: + mainModules = loadFeatureMainModules() + for featureName, module in mainModules.items(): + if hasattr(module, "onStart"): + try: + await module.onStart(eventUser) + logger.info(f"Feature '{featureName}' started") + except Exception as e: + logger.error(f"Feature '{featureName}' failed to start: {e}") + except Exception as e: + logger.warning(f"Could not initialize feature containers: {e}") + # --- Init Managers --- - await featuresLifecycle.start(eventUser) + await subAutomationSchedule.start(eventUser) # Automation scheduler eventManager.start() # Register audit log cleanup scheduler @@ -304,7 +314,21 @@ async def lifespan(app: FastAPI): # --- Stop Managers --- eventManager.stop() - await featuresLifecycle.stop(eventUser) + await subAutomationSchedule.stop(eventUser) # Automation scheduler + + # --- Stop Feature Containers (Plug&Play) --- + try: + mainModules = loadFeatureMainModules() + for featureName, module in mainModules.items(): + if hasattr(module, "onStop"): + try: + await module.onStop(eventUser) + logger.info(f"Feature '{featureName}' stopped") + except Exception as e: + logger.error(f"Feature '{featureName}' failed to stop: {e}") + except Exception as e: + logger.warning(f"Could not shutdown feature containers: {e}") + logger.info("Application has been shut down") @@ -412,9 +436,6 @@ app.include_router(userRouter) from modules.routes.routeDataFiles import router as fileRouter app.include_router(fileRouter) -from modules.routes.routeFeatureNeutralization import router as neutralizationRouter -app.include_router(neutralizationRouter) - from modules.routes.routeDataPrompts import router as promptRouter app.include_router(promptRouter) @@ -424,12 +445,6 @@ app.include_router(connectionsRouter) from modules.routes.routeDataWorkflows import router as dataWorkflowsRouter app.include_router(dataWorkflowsRouter) -from modules.routes.routeFeatureChatDynamic import router as chatPlaygroundRouter -app.include_router(chatPlaygroundRouter) - -from modules.routes.routeFeatureRealEstate import router as realEstateRouter -app.include_router(realEstateRouter) - from modules.routes.routeSecurityLocal import router as localRouter app.include_router(localRouter) @@ -448,27 +463,15 @@ app.include_router(adminSecurityRouter) from modules.routes.routeSharepoint import router as sharepointRouter app.include_router(sharepointRouter) -from modules.routes.routeDataAutomation import router as automationRouter -app.include_router(automationRouter) - from modules.routes.routeAdminAutomationEvents import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) from modules.routes.routeAdminRbacRules import router as rbacAdminRulesRouter app.include_router(rbacAdminRulesRouter) -from modules.routes.routeOptions import router as optionsRouter -app.include_router(optionsRouter) - from modules.routes.routeMessaging import router as messagingRouter app.include_router(messagingRouter) -from modules.routes.routeFeatureChatbot import router as chatbotRouter -app.include_router(chatbotRouter) - -from modules.routes.routeFeatureTrustee import router as trusteeRouter -app.include_router(trusteeRouter) - # Phase 8: New Feature Routes from modules.routes.routeAdminFeatures import router as featuresAdminRouter app.include_router(featuresAdminRouter) @@ -481,3 +484,12 @@ app.include_router(rbacAdminExportRouter) from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) + +# ============================================================================ +# PLUG&PLAY FEATURE ROUTERS +# Dynamically load routers from feature containers in modules/features/ +# ============================================================================ +from modules.features.featureRegistry import loadFeatureRouters + +featureLoadResults = loadFeatureRouters(app) +logger.info(f"Feature router load results: {featureLoadResults}") diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py index b884382c..1a1e49e8 100644 --- a/modules/datamodels/datamodelWorkflow.py +++ b/modules/datamodels/datamodelWorkflow.py @@ -12,12 +12,6 @@ from modules.shared.jsonUtils import extractJsonString, tryParseJson, repairBrok # Import DocumentReferenceList at runtime (needed for ActionDefinition) from modules.datamodels.datamodelDocref import DocumentReferenceList -# Forward references for circular imports (use string annotations) -if TYPE_CHECKING: - from modules.datamodels.datamodelChat import ChatDocument, ActionResult - from modules.datamodels.datamodelExtraction import ExtractionOptions - - class ActionDefinition(BaseModel): """Action definition with selection and parameters from planning phase""" diff --git a/modules/aicore/aicoreBase.py b/modules/features/aichat/aicore/aicoreBase.py similarity index 100% rename from modules/aicore/aicoreBase.py rename to modules/features/aichat/aicore/aicoreBase.py diff --git a/modules/aicore/aicoreModelRegistry.py b/modules/features/aichat/aicore/aicoreModelRegistry.py similarity index 99% rename from modules/aicore/aicoreModelRegistry.py rename to modules/features/aichat/aicore/aicoreModelRegistry.py index ed82572d..8fd0e284 100644 --- a/modules/aicore/aicoreModelRegistry.py +++ b/modules/features/aichat/aicore/aicoreModelRegistry.py @@ -10,7 +10,7 @@ import importlib import os from typing import Dict, List, Optional, Any from modules.datamodels.datamodelAi import AiModel -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelUam import User from modules.security.rbacHelpers import checkResourceAccess from modules.security.rbac import RbacClass diff --git a/modules/aicore/aicoreModelSelector.py b/modules/features/aichat/aicore/aicoreModelSelector.py similarity index 100% rename from modules/aicore/aicoreModelSelector.py rename to modules/features/aichat/aicore/aicoreModelSelector.py diff --git a/modules/aicore/aicorePluginAnthropic.py b/modules/features/aichat/aicore/aicorePluginAnthropic.py similarity index 99% rename from modules/aicore/aicorePluginAnthropic.py rename to modules/features/aichat/aicore/aicorePluginAnthropic.py index 2a619056..0d80aeaa 100644 --- a/modules/aicore/aicorePluginAnthropic.py +++ b/modules/features/aichat/aicore/aicorePluginAnthropic.py @@ -6,7 +6,7 @@ import os from typing import Dict, Any, List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings # Configure logger diff --git a/modules/aicore/aicorePluginInternal.py b/modules/features/aichat/aicore/aicorePluginInternal.py similarity index 98% rename from modules/aicore/aicorePluginInternal.py rename to modules/features/aichat/aicore/aicorePluginInternal.py index f8e13e16..1b73c27e 100644 --- a/modules/aicore/aicorePluginInternal.py +++ b/modules/features/aichat/aicore/aicorePluginInternal.py @@ -2,7 +2,7 @@ # All rights reserved. import logging from typing import List -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings # Configure logger diff --git a/modules/aicore/aicorePluginOpenai.py b/modules/features/aichat/aicore/aicorePluginOpenai.py similarity index 99% rename from modules/aicore/aicorePluginOpenai.py rename to modules/features/aichat/aicore/aicorePluginOpenai.py index 932586e6..711f0c35 100644 --- a/modules/aicore/aicorePluginOpenai.py +++ b/modules/features/aichat/aicore/aicorePluginOpenai.py @@ -5,7 +5,7 @@ import httpx from typing import List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings # Configure logger diff --git a/modules/aicore/aicorePluginPerplexity.py b/modules/features/aichat/aicore/aicorePluginPerplexity.py similarity index 99% rename from modules/aicore/aicorePluginPerplexity.py rename to modules/features/aichat/aicore/aicorePluginPerplexity.py index e129b047..f537d83c 100644 --- a/modules/aicore/aicorePluginPerplexity.py +++ b/modules/features/aichat/aicore/aicorePluginPerplexity.py @@ -5,7 +5,7 @@ import httpx from typing import List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings, AiCallPromptWebSearch, AiCallPromptWebCrawl from modules.datamodels.datamodelTools import CountryCodes diff --git a/modules/aicore/aicorePluginTavily.py b/modules/features/aichat/aicore/aicorePluginTavily.py similarity index 99% rename from modules/aicore/aicorePluginTavily.py rename to modules/features/aichat/aicore/aicorePluginTavily.py index 1a5cbc65..1d2ece75 100644 --- a/modules/aicore/aicorePluginTavily.py +++ b/modules/features/aichat/aicore/aicorePluginTavily.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from typing import Optional, List, Dict from tavily import AsyncTavilyClient from modules.shared.configuration import APP_CONFIG -from modules.aicore.aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings, AiCallPromptWebSearch, AiCallPromptWebCrawl from modules.datamodels.datamodelTools import CountryCodes diff --git a/modules/datamodels/datamodelChat.py b/modules/features/aichat/datamodelFeatureAiChat.py similarity index 100% rename from modules/datamodels/datamodelChat.py rename to modules/features/aichat/datamodelFeatureAiChat.py diff --git a/modules/interfaces/interfaceDbChat.py b/modules/features/aichat/interfaceFeatureAiChat.py similarity index 97% rename from modules/interfaces/interfaceDbChat.py rename to modules/features/aichat/interfaceFeatureAiChat.py index d171c2ca..de3d42d1 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/features/aichat/interfaceFeatureAiChat.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChat import ( +from .datamodelFeatureAiChat import ( ChatDocument, ChatStat, ChatLog, @@ -1095,26 +1095,6 @@ class ChatObjects: actionName=createdMessage.get("actionName") ) - # Emit message event for streaming (if event manager is available) - try: - from modules.features.chatbot.eventManager import get_event_manager - event_manager = get_event_manager() - message_timestamp = parseTimestamp(chat_message.publishedAt, default=getUtcTimestamp()) - # Emit message event in exact chatData format: {type, createdAt, item} - asyncio.create_task(event_manager.emit_event( - context_id=workflowId, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": chat_message.dict() - }, - event_category="chat" - )) - except Exception as e: - # Event manager not available or error - continue without emitting - logger.debug(f"Could not emit message event: {e}") - # Debug: Store message and documents for debugging - only if debug enabled storeDebugMessageAndDocuments(chat_message, self.currentUser) @@ -1481,29 +1461,6 @@ class ChatObjects: # Create log in normalized table createdLog = self.db.recordCreate(ChatLog, log_model) - # Emit log event for streaming (only for chatbot workflows) - # Only emit events for chatbot workflows, not for automation or dynamic workflows - if workflow.workflowMode == WorkflowModeEnum.WORKFLOW_CHATBOT: - try: - from modules.features.chatbot.eventManager import get_event_manager - event_manager = get_event_manager() - log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) - # Emit log event in exact chatData format: {type, createdAt, item} - asyncio.create_task(event_manager.emit_event( - workflowId, - "chatdata", - "New log", - "log", - { - "type": "log", - "createdAt": log_timestamp, - "item": ChatLog(**createdLog).model_dump() - } - )) - except Exception as e: - # Event manager not available or error - continue without emitting - logger.debug(f"Could not emit log event: {e}") - # Return validated ChatLog instance return ChatLog(**createdLog) @@ -1888,14 +1845,6 @@ class ChatObjects: if not self.checkRbacPermission(AutomationDefinition, "delete", automationId): raise PermissionError(f"No permission to delete automation {automationId}") - # Remove event if exists - if existing.get("eventId"): - from modules.shared.eventManagement import eventManager - try: - eventManager.remove(existing["eventId"]) - except Exception as e: - logger.warning(f"Error removing event {existing['eventId']}: {str(e)}") - # Delete automation from database self.db.recordDelete(AutomationDefinition, automationId) diff --git a/modules/features/aichat/mainAiChat.py b/modules/features/aichat/mainAiChat.py new file mode 100644 index 00000000..fbd6b91a --- /dev/null +++ b/modules/features/aichat/mainAiChat.py @@ -0,0 +1,166 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +AIChat Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. + +AIChat is the dynamic chat workflow feature that handles: +- AI-powered document processing +- Dynamic workflow execution +- Automation definitions +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "chatworkflow" +FEATURE_LABEL = {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"} +FEATURE_ICON = "mdi-message-cog" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.aichat.workflows", + "label": {"en": "Workflows", "de": "Workflows", "fr": "Workflows"}, + "meta": {"area": "workflows"} + }, + { + "objectKey": "ui.feature.aichat.automations", + "label": {"en": "Automations", "de": "Automatisierungen", "fr": "Automatisations"}, + "meta": {"area": "automations"} + }, + { + "objectKey": "ui.feature.aichat.logs", + "label": {"en": "Logs", "de": "Logs", "fr": "Journaux"}, + "meta": {"area": "logs"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.aichat.workflow.start", + "label": {"en": "Start Workflow", "de": "Workflow starten", "fr": "Démarrer workflow"}, + "meta": {"endpoint": "/api/chat/playground/start", "method": "POST"} + }, + { + "objectKey": "resource.feature.aichat.workflow.stop", + "label": {"en": "Stop Workflow", "de": "Workflow stoppen", "fr": "Arrêter workflow"}, + "meta": {"endpoint": "/api/chat/playground/stop/{workflowId}", "method": "POST"} + }, + { + "objectKey": "resource.feature.aichat.workflow.delete", + "label": {"en": "Delete Workflow", "de": "Workflow löschen", "fr": "Supprimer workflow"}, + "meta": {"endpoint": "/api/chat/playground/workflow/{workflowId}", "method": "DELETE"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "workflow-admin", + "description": { + "en": "Workflow Administrator - Full access to workflow configuration and execution", + "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", + "fr": "Administrateur workflow - Accès complet à la configuration et exécution" + } + }, + { + "roleLabel": "workflow-editor", + "description": { + "en": "Workflow Editor - Create and modify workflows", + "de": "Workflow-Editor - Workflows erstellen und bearbeiten", + "fr": "Éditeur workflow - Créer et modifier les workflows" + } + }, + { + "roleLabel": "workflow-viewer", + "description": { + "en": "Workflow Viewer - View workflows and execution results", + "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", + "fr": "Visualiseur workflow - Consulter les workflows et résultats" + } + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """ + Register this feature's RBAC objects in the catalog. + + Args: + catalogService: The RBAC catalog service instance + + Returns: + True if registration was successful + """ + try: + # Register UI objects + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + # Register Resource objects + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False + + +async def onStart(eventUser) -> None: + """ + Called when the feature container starts. + Initializes AI connectors for model registry. + """ + try: + from .aicore.aicoreModelRegistry import modelRegistry + modelRegistry.ensureConnectorsRegistered() + logger.info(f"Feature '{FEATURE_CODE}' started - AI connectors initialized") + except Exception as e: + logger.error(f"Feature '{FEATURE_CODE}' failed to initialize AI connectors: {e}") + + +async def onStop(eventUser) -> None: + """Called when the feature container stops.""" + logger.info(f"Feature '{FEATURE_CODE}' stopped") diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/features/aichat/routeFeatureAiChat.py similarity index 95% rename from modules/routes/routeFeatureChatDynamic.py rename to modules/features/aichat/routeFeatureAiChat.py index 5be544a8..3eaaf624 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/features/aichat/routeFeatureAiChat.py @@ -13,13 +13,13 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChat as interfaceDbChat +from . import interfaceFeatureAiChat as interfaceDbChat # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from .datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions -from modules.features.workflow import chatStart, chatStop +from modules.workflows.automation import chatStart, chatStop # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/features/aichat/serviceAi/mainServiceAi.py similarity index 97% rename from modules/services/serviceAi/mainServiceAi.py rename to modules/features/aichat/serviceAi/mainServiceAi.py index cd86c6a8..e5d2605f 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/features/aichat/serviceAi/mainServiceAi.py @@ -6,8 +6,8 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument -from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.features.aichat.datamodelFeatureAiChat import PromptPlaceholder, ChatDocument +from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData @@ -16,7 +16,7 @@ from modules.interfaces.interfaceAiObjects import AiObjects from modules.shared.jsonUtils import ( parseJsonWithModel ) -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler +from .subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState logger = logging.getLogger(__name__) @@ -49,12 +49,12 @@ class AiService: self.extractionService = ExtractionService(self.services) # Initialize new submodules - from modules.services.serviceAi.subResponseParsing import ResponseParser - from modules.services.serviceAi.subDocumentIntents import DocumentIntentAnalyzer - from modules.services.serviceAi.subContentExtraction import ContentExtractor - from modules.services.serviceAi.subStructureGeneration import StructureGenerator - from modules.services.serviceAi.subStructureFilling import StructureFiller - from modules.services.serviceAi.subAiCallLooping import AiCallLooper + from .subResponseParsing import ResponseParser + from .subDocumentIntents import DocumentIntentAnalyzer + from .subContentExtraction import ContentExtractor + from .subStructureGeneration import StructureGenerator + from .subStructureFilling import StructureFiller + from .subAiCallLooping import AiCallLooper if not hasattr(self, 'responseParser'): logger.info("Initializing ResponseParser...") @@ -329,7 +329,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle IMAGE_GENERATE operation type using image generation path.""" - from modules.services.serviceGeneration.paths.imagePath import ImageGenerationPath + from modules.features.aichat.serviceGeneration.paths.imagePath import ImageGenerationPath imagePath = ImageGenerationPath(self.services) @@ -514,7 +514,7 @@ Respond with ONLY a JSON object in this exact format: ) try: - from modules.services.serviceGeneration.mainServiceGeneration import GenerationService + from modules.features.aichat.serviceGeneration.mainServiceGeneration import GenerationService generationService = GenerationService(self.services) @@ -829,7 +829,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle code generation using code generation path.""" - from modules.services.serviceGeneration.paths.codePath import CodeGenerationPath + from modules.features.aichat.serviceGeneration.paths.codePath import CodeGenerationPath codePath = CodeGenerationPath(self.services) return await codePath.generateCode( @@ -852,7 +852,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle document generation using document generation path.""" - from modules.services.serviceGeneration.paths.documentPath import DocumentGenerationPath + from modules.features.aichat.serviceGeneration.paths.documentPath import DocumentGenerationPath # Set compression options for document generation options.compressPrompt = False diff --git a/modules/services/serviceAi/merge_1.txt b/modules/features/aichat/serviceAi/merge_1.txt similarity index 100% rename from modules/services/serviceAi/merge_1.txt rename to modules/features/aichat/serviceAi/merge_1.txt diff --git a/modules/services/serviceAi/subAiCallLooping-flow.md b/modules/features/aichat/serviceAi/subAiCallLooping-flow.md similarity index 100% rename from modules/services/serviceAi/subAiCallLooping-flow.md rename to modules/features/aichat/serviceAi/subAiCallLooping-flow.md diff --git a/modules/services/serviceAi/subAiCallLooping.py b/modules/features/aichat/serviceAi/subAiCallLooping.py similarity index 99% rename from modules/services/serviceAi/subAiCallLooping.py rename to modules/features/aichat/serviceAi/subAiCallLooping.py index 6427b2e0..5f3fb79f 100644 --- a/modules/services/serviceAi/subAiCallLooping.py +++ b/modules/features/aichat/serviceAi/subAiCallLooping.py @@ -53,8 +53,8 @@ from modules.datamodels.datamodelAi import ( AiCallRequest, AiCallOptions ) from modules.datamodels.datamodelExtraction import ContentPart -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler -from modules.services.serviceAi.subLoopingUseCases import LoopingUseCaseRegistry +from .subJsonResponseHandling import JsonResponseHandler +from .subLoopingUseCases import LoopingUseCaseRegistry from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.jsonContinuation import getContexts from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/features/aichat/serviceAi/subContentExtraction.py similarity index 99% rename from modules/services/serviceAi/subContentExtraction.py rename to modules/features/aichat/serviceAi/subContentExtraction.py index a866f68f..cace339d 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/features/aichat/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.features.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/features/aichat/serviceAi/subDocumentIntents.py similarity index 99% rename from modules/services/serviceAi/subDocumentIntents.py rename to modules/features/aichat/serviceAi/subDocumentIntents.py index 821851a4..2d45179a 100644 --- a/modules/services/serviceAi/subDocumentIntents.py +++ b/modules/features/aichat/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.features.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subJsonMerger.py b/modules/features/aichat/serviceAi/subJsonMerger.py similarity index 100% rename from modules/services/serviceAi/subJsonMerger.py rename to modules/features/aichat/serviceAi/subJsonMerger.py diff --git a/modules/services/serviceAi/subJsonResponseHandling.py b/modules/features/aichat/serviceAi/subJsonResponseHandling.py similarity index 99% rename from modules/services/serviceAi/subJsonResponseHandling.py rename to modules/features/aichat/serviceAi/subJsonResponseHandling.py index c088d598..b2cfe084 100644 --- a/modules/services/serviceAi/subJsonResponseHandling.py +++ b/modules/features/aichat/serviceAi/subJsonResponseHandling.py @@ -1346,7 +1346,7 @@ class JsonResponseHandler: # Use new modular merger try: - from modules.services.serviceAi.subJsonMerger import ModularJsonMerger + from .subJsonMerger import ModularJsonMerger result, hasOverlap = ModularJsonMerger.merge(accumulated, newFragment) # IMPORTANT: ModularJsonMerger returns unclosed JSON if overlap found (with incomplete element at end) # If no overlap, returns closed JSON (iterations should stop) diff --git a/modules/services/serviceAi/subLoopingUseCases.py b/modules/features/aichat/serviceAi/subLoopingUseCases.py similarity index 100% rename from modules/services/serviceAi/subLoopingUseCases.py rename to modules/features/aichat/serviceAi/subLoopingUseCases.py diff --git a/modules/services/serviceAi/subResponseParsing.py b/modules/features/aichat/serviceAi/subResponseParsing.py similarity index 99% rename from modules/services/serviceAi/subResponseParsing.py rename to modules/features/aichat/serviceAi/subResponseParsing.py index a2d568d9..68c123ac 100644 --- a/modules/services/serviceAi/subResponseParsing.py +++ b/modules/features/aichat/serviceAi/subResponseParsing.py @@ -15,7 +15,7 @@ import logging from typing import Dict, Any, List, Optional, Tuple from modules.shared.jsonUtils import extractJsonString, repairBrokenJson, extractSectionsFromDocument -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler +from .subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState logger = logging.getLogger(__name__) diff --git a/modules/services/serviceAi/subStructureFilling.py b/modules/features/aichat/serviceAi/subStructureFilling.py similarity index 99% rename from modules/services/serviceAi/subStructureFilling.py rename to modules/features/aichat/serviceAi/subStructureFilling.py index 5145ad54..c5ba2f8a 100644 --- a/modules/services/serviceAi/subStructureFilling.py +++ b/modules/features/aichat/serviceAi/subStructureFilling.py @@ -2531,7 +2531,7 @@ CRITICAL: List of accepted section content types (e.g., ["table", "code_block"]) """ try: - from modules.services.serviceGeneration.renderers.registry import getRenderer + from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer # Get renderer for this format renderer = getRenderer(outputFormat, self.services) diff --git a/modules/services/serviceAi/subStructureGeneration.py b/modules/features/aichat/serviceAi/subStructureGeneration.py similarity index 99% rename from modules/services/serviceAi/subStructureGeneration.py rename to modules/features/aichat/serviceAi/subStructureGeneration.py index 64624b84..ff01c3dd 100644 --- a/modules/services/serviceAi/subStructureGeneration.py +++ b/modules/features/aichat/serviceAi/subStructureGeneration.py @@ -231,7 +231,7 @@ CRITICAL: raise ValueError("Structure has no documents - cannot generate without documents") # Import renderer registry for format validation (existing infrastructure) - from modules.services.serviceGeneration.renderers.registry import getRenderer + from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer # Validate and fix each document for doc in documents: diff --git a/modules/services/serviceExtraction/__init__.py b/modules/features/aichat/serviceExtraction/__init__.py similarity index 100% rename from modules/services/serviceExtraction/__init__.py rename to modules/features/aichat/serviceExtraction/__init__.py diff --git a/modules/services/serviceExtraction/chunking/__init__.py b/modules/features/aichat/serviceExtraction/chunking/__init__.py similarity index 100% rename from modules/services/serviceExtraction/chunking/__init__.py rename to modules/features/aichat/serviceExtraction/chunking/__init__.py diff --git a/modules/services/serviceExtraction/chunking/chunkerImage.py b/modules/features/aichat/serviceExtraction/chunking/chunkerImage.py similarity index 100% rename from modules/services/serviceExtraction/chunking/chunkerImage.py rename to modules/features/aichat/serviceExtraction/chunking/chunkerImage.py diff --git a/modules/services/serviceExtraction/chunking/chunkerStructure.py b/modules/features/aichat/serviceExtraction/chunking/chunkerStructure.py similarity index 100% rename from modules/services/serviceExtraction/chunking/chunkerStructure.py rename to modules/features/aichat/serviceExtraction/chunking/chunkerStructure.py diff --git a/modules/services/serviceExtraction/chunking/chunkerTable.py b/modules/features/aichat/serviceExtraction/chunking/chunkerTable.py similarity index 100% rename from modules/services/serviceExtraction/chunking/chunkerTable.py rename to modules/features/aichat/serviceExtraction/chunking/chunkerTable.py diff --git a/modules/services/serviceExtraction/chunking/chunkerText.py b/modules/features/aichat/serviceExtraction/chunking/chunkerText.py similarity index 100% rename from modules/services/serviceExtraction/chunking/chunkerText.py rename to modules/features/aichat/serviceExtraction/chunking/chunkerText.py diff --git a/modules/services/serviceExtraction/extractors/__init__.py b/modules/features/aichat/serviceExtraction/extractors/__init__.py similarity index 100% rename from modules/services/serviceExtraction/extractors/__init__.py rename to modules/features/aichat/serviceExtraction/extractors/__init__.py diff --git a/modules/services/serviceExtraction/extractors/extractorBinary.py b/modules/features/aichat/serviceExtraction/extractors/extractorBinary.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorBinary.py rename to modules/features/aichat/serviceExtraction/extractors/extractorBinary.py diff --git a/modules/services/serviceExtraction/extractors/extractorCsv.py b/modules/features/aichat/serviceExtraction/extractors/extractorCsv.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorCsv.py rename to modules/features/aichat/serviceExtraction/extractors/extractorCsv.py diff --git a/modules/services/serviceExtraction/extractors/extractorDocx.py b/modules/features/aichat/serviceExtraction/extractors/extractorDocx.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorDocx.py rename to modules/features/aichat/serviceExtraction/extractors/extractorDocx.py diff --git a/modules/services/serviceExtraction/extractors/extractorHtml.py b/modules/features/aichat/serviceExtraction/extractors/extractorHtml.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorHtml.py rename to modules/features/aichat/serviceExtraction/extractors/extractorHtml.py diff --git a/modules/services/serviceExtraction/extractors/extractorImage.py b/modules/features/aichat/serviceExtraction/extractors/extractorImage.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorImage.py rename to modules/features/aichat/serviceExtraction/extractors/extractorImage.py diff --git a/modules/services/serviceExtraction/extractors/extractorJson.py b/modules/features/aichat/serviceExtraction/extractors/extractorJson.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorJson.py rename to modules/features/aichat/serviceExtraction/extractors/extractorJson.py diff --git a/modules/services/serviceExtraction/extractors/extractorPdf.py b/modules/features/aichat/serviceExtraction/extractors/extractorPdf.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorPdf.py rename to modules/features/aichat/serviceExtraction/extractors/extractorPdf.py diff --git a/modules/services/serviceExtraction/extractors/extractorPptx.py b/modules/features/aichat/serviceExtraction/extractors/extractorPptx.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorPptx.py rename to modules/features/aichat/serviceExtraction/extractors/extractorPptx.py diff --git a/modules/services/serviceExtraction/extractors/extractorSql.py b/modules/features/aichat/serviceExtraction/extractors/extractorSql.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorSql.py rename to modules/features/aichat/serviceExtraction/extractors/extractorSql.py diff --git a/modules/services/serviceExtraction/extractors/extractorText.py b/modules/features/aichat/serviceExtraction/extractors/extractorText.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorText.py rename to modules/features/aichat/serviceExtraction/extractors/extractorText.py diff --git a/modules/services/serviceExtraction/extractors/extractorXlsx.py b/modules/features/aichat/serviceExtraction/extractors/extractorXlsx.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorXlsx.py rename to modules/features/aichat/serviceExtraction/extractors/extractorXlsx.py diff --git a/modules/services/serviceExtraction/extractors/extractorXml.py b/modules/features/aichat/serviceExtraction/extractors/extractorXml.py similarity index 100% rename from modules/services/serviceExtraction/extractors/extractorXml.py rename to modules/features/aichat/serviceExtraction/extractors/extractorXml.py diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/features/aichat/serviceExtraction/mainServiceExtraction.py similarity index 99% rename from modules/services/serviceExtraction/mainServiceExtraction.py rename to modules/features/aichat/serviceExtraction/mainServiceExtraction.py index 199201eb..abe32352 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/features/aichat/serviceExtraction/mainServiceExtraction.py @@ -11,10 +11,10 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.datamodels.datamodelChat import ChatDocument +from modules.features.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall -from modules.aicore.aicoreModelRegistry import modelRegistry -from modules.aicore.aicoreModelSelector import modelSelector +from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.features.aichat.aicore.aicoreModelSelector import modelSelector from modules.shared.jsonUtils import stripCodeFences diff --git a/modules/services/serviceExtraction/merging/__init__.py b/modules/features/aichat/serviceExtraction/merging/__init__.py similarity index 100% rename from modules/services/serviceExtraction/merging/__init__.py rename to modules/features/aichat/serviceExtraction/merging/__init__.py diff --git a/modules/services/serviceExtraction/merging/mergerDefault.py b/modules/features/aichat/serviceExtraction/merging/mergerDefault.py similarity index 100% rename from modules/services/serviceExtraction/merging/mergerDefault.py rename to modules/features/aichat/serviceExtraction/merging/mergerDefault.py diff --git a/modules/services/serviceExtraction/merging/mergerTable.py b/modules/features/aichat/serviceExtraction/merging/mergerTable.py similarity index 100% rename from modules/services/serviceExtraction/merging/mergerTable.py rename to modules/features/aichat/serviceExtraction/merging/mergerTable.py diff --git a/modules/services/serviceExtraction/merging/mergerText.py b/modules/features/aichat/serviceExtraction/merging/mergerText.py similarity index 100% rename from modules/services/serviceExtraction/merging/mergerText.py rename to modules/features/aichat/serviceExtraction/merging/mergerText.py diff --git a/modules/services/serviceExtraction/subMerger.py b/modules/features/aichat/serviceExtraction/subMerger.py similarity index 100% rename from modules/services/serviceExtraction/subMerger.py rename to modules/features/aichat/serviceExtraction/subMerger.py diff --git a/modules/services/serviceExtraction/subPipeline.py b/modules/features/aichat/serviceExtraction/subPipeline.py similarity index 100% rename from modules/services/serviceExtraction/subPipeline.py rename to modules/features/aichat/serviceExtraction/subPipeline.py diff --git a/modules/services/serviceExtraction/subPromptBuilderExtraction.py b/modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py similarity index 98% rename from modules/services/serviceExtraction/subPromptBuilderExtraction.py rename to modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py index 8f8f756d..50228983 100644 --- a/modules/services/serviceExtraction/subPromptBuilderExtraction.py +++ b/modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py @@ -13,7 +13,7 @@ from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, Operati # Type hint for renderer parameter from typing import TYPE_CHECKING if TYPE_CHECKING: - from modules.services.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer + from modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer _RendererLike = BaseRenderer else: _RendererLike = Any diff --git a/modules/services/serviceExtraction/subRegistry.py b/modules/features/aichat/serviceExtraction/subRegistry.py similarity index 99% rename from modules/services/serviceExtraction/subRegistry.py rename to modules/features/aichat/serviceExtraction/subRegistry.py index 32727746..972c1eb7 100644 --- a/modules/services/serviceExtraction/subRegistry.py +++ b/modules/features/aichat/serviceExtraction/subRegistry.py @@ -71,7 +71,7 @@ class ExtractorRegistry: module_name = file_path.stem try: # Import the module - module = importlib.import_module(f".{module_name}", package="modules.services.serviceExtraction.extractors") + module = importlib.import_module(f".{module_name}", package="modules.features.aichat.serviceExtraction.extractors") # Find all extractor classes in the module for attr_name in dir(module): diff --git a/modules/services/serviceExtraction/subUtils.py b/modules/features/aichat/serviceExtraction/subUtils.py similarity index 100% rename from modules/services/serviceExtraction/subUtils.py rename to modules/features/aichat/serviceExtraction/subUtils.py diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/features/aichat/serviceGeneration/mainServiceGeneration.py similarity index 98% rename from modules/services/serviceGeneration/mainServiceGeneration.py rename to modules/features/aichat/serviceGeneration/mainServiceGeneration.py index a49b78c7..8aadd081 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/features/aichat/serviceGeneration/mainServiceGeneration.py @@ -6,8 +6,8 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.datamodels.datamodelChat import ChatDocument -from modules.services.serviceGeneration.subDocumentUtility import ( +from modules.features.aichat.datamodelFeatureAiChat import ChatDocument +from modules.features.aichat.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, detectMimeTypeFromContent, @@ -414,7 +414,7 @@ class GenerationService: continue # Check output style classification (code/document/image/etc.) from renderer - from modules.services.serviceGeneration.renderers.registry import getOutputStyle + from modules.features.aichat.serviceGeneration.renderers.registry import getOutputStyle outputStyle = getOutputStyle(docFormat) if outputStyle: logger.debug(f"Document {doc.get('id', docIndex)} format '{docFormat}' classified as '{outputStyle}' style") @@ -471,8 +471,8 @@ class GenerationService: Complete document structure with populated elements ready for rendering """ try: - from modules.services.serviceGeneration.subStructureGenerator import StructureGenerator - from modules.services.serviceGeneration.subContentGenerator import ContentGenerator + from modules.features.aichat.serviceGeneration.subStructureGenerator import StructureGenerator + from modules.features.aichat.serviceGeneration.subContentGenerator import ContentGenerator # Phase 1: Generate structure skeleton if progressCallback: @@ -537,7 +537,7 @@ class GenerationService: aiService=None ) -> str: """Get adaptive extraction prompt.""" - from modules.services.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt + from modules.features.aichat.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt return await buildExtractionPrompt( outputFormat=outputFormat, userPrompt=userPrompt, diff --git a/modules/services/serviceGeneration/paths/codePath.py b/modules/features/aichat/serviceGeneration/paths/codePath.py similarity index 99% rename from modules/services/serviceGeneration/paths/codePath.py rename to modules/features/aichat/serviceGeneration/paths/codePath.py index f2470385..5e151066 100644 --- a/modules/services/serviceGeneration/paths/codePath.py +++ b/modules/features/aichat/serviceGeneration/paths/codePath.py @@ -920,7 +920,7 @@ CRITICAL: def _getCodeRenderer(self, fileType: str): """Get code renderer for file type.""" - from modules.services.serviceGeneration.renderers.registry import getRenderer + from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer # Map file types to renderer formats formatMap = { diff --git a/modules/services/serviceGeneration/paths/documentPath.py b/modules/features/aichat/serviceGeneration/paths/documentPath.py similarity index 100% rename from modules/services/serviceGeneration/paths/documentPath.py rename to modules/features/aichat/serviceGeneration/paths/documentPath.py diff --git a/modules/services/serviceGeneration/paths/imagePath.py b/modules/features/aichat/serviceGeneration/paths/imagePath.py similarity index 100% rename from modules/services/serviceGeneration/paths/imagePath.py rename to modules/features/aichat/serviceGeneration/paths/imagePath.py diff --git a/modules/services/serviceGeneration/renderers/codeRendererBaseTemplate.py b/modules/features/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py similarity index 100% rename from modules/services/serviceGeneration/renderers/codeRendererBaseTemplate.py rename to modules/features/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py diff --git a/modules/services/serviceGeneration/renderers/documentRendererBaseTemplate.py b/modules/features/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py similarity index 100% rename from modules/services/serviceGeneration/renderers/documentRendererBaseTemplate.py rename to modules/features/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py diff --git a/modules/services/serviceGeneration/renderers/registry.py b/modules/features/aichat/serviceGeneration/renderers/registry.py similarity index 100% rename from modules/services/serviceGeneration/renderers/registry.py rename to modules/features/aichat/serviceGeneration/renderers/registry.py diff --git a/modules/services/serviceGeneration/renderers/rendererCodeCsv.py b/modules/features/aichat/serviceGeneration/renderers/rendererCodeCsv.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererCodeCsv.py rename to modules/features/aichat/serviceGeneration/renderers/rendererCodeCsv.py diff --git a/modules/services/serviceGeneration/renderers/rendererCodeJson.py b/modules/features/aichat/serviceGeneration/renderers/rendererCodeJson.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererCodeJson.py rename to modules/features/aichat/serviceGeneration/renderers/rendererCodeJson.py diff --git a/modules/services/serviceGeneration/renderers/rendererCodeXml.py b/modules/features/aichat/serviceGeneration/renderers/rendererCodeXml.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererCodeXml.py rename to modules/features/aichat/serviceGeneration/renderers/rendererCodeXml.py diff --git a/modules/services/serviceGeneration/renderers/rendererCsv.py b/modules/features/aichat/serviceGeneration/renderers/rendererCsv.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererCsv.py rename to modules/features/aichat/serviceGeneration/renderers/rendererCsv.py diff --git a/modules/services/serviceGeneration/renderers/rendererDocx.py b/modules/features/aichat/serviceGeneration/renderers/rendererDocx.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererDocx.py rename to modules/features/aichat/serviceGeneration/renderers/rendererDocx.py diff --git a/modules/services/serviceGeneration/renderers/rendererHtml.py b/modules/features/aichat/serviceGeneration/renderers/rendererHtml.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererHtml.py rename to modules/features/aichat/serviceGeneration/renderers/rendererHtml.py diff --git a/modules/services/serviceGeneration/renderers/rendererImage.py b/modules/features/aichat/serviceGeneration/renderers/rendererImage.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererImage.py rename to modules/features/aichat/serviceGeneration/renderers/rendererImage.py diff --git a/modules/services/serviceGeneration/renderers/rendererJson.py b/modules/features/aichat/serviceGeneration/renderers/rendererJson.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererJson.py rename to modules/features/aichat/serviceGeneration/renderers/rendererJson.py diff --git a/modules/services/serviceGeneration/renderers/rendererMarkdown.py b/modules/features/aichat/serviceGeneration/renderers/rendererMarkdown.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererMarkdown.py rename to modules/features/aichat/serviceGeneration/renderers/rendererMarkdown.py diff --git a/modules/services/serviceGeneration/renderers/rendererPdf.py b/modules/features/aichat/serviceGeneration/renderers/rendererPdf.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererPdf.py rename to modules/features/aichat/serviceGeneration/renderers/rendererPdf.py diff --git a/modules/services/serviceGeneration/renderers/rendererPptx.py b/modules/features/aichat/serviceGeneration/renderers/rendererPptx.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererPptx.py rename to modules/features/aichat/serviceGeneration/renderers/rendererPptx.py diff --git a/modules/services/serviceGeneration/renderers/rendererText.py b/modules/features/aichat/serviceGeneration/renderers/rendererText.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererText.py rename to modules/features/aichat/serviceGeneration/renderers/rendererText.py diff --git a/modules/services/serviceGeneration/renderers/rendererXlsx.py b/modules/features/aichat/serviceGeneration/renderers/rendererXlsx.py similarity index 100% rename from modules/services/serviceGeneration/renderers/rendererXlsx.py rename to modules/features/aichat/serviceGeneration/renderers/rendererXlsx.py diff --git a/modules/services/serviceGeneration/subContentGenerator.py b/modules/features/aichat/serviceGeneration/subContentGenerator.py similarity index 99% rename from modules/services/serviceGeneration/subContentGenerator.py rename to modules/features/aichat/serviceGeneration/subContentGenerator.py index 86464ef6..5995077f 100644 --- a/modules/services/serviceGeneration/subContentGenerator.py +++ b/modules/features/aichat/serviceGeneration/subContentGenerator.py @@ -12,7 +12,7 @@ import base64 import re import traceback from typing import Dict, Any, Optional, List, Callable -from modules.services.serviceGeneration.subContentIntegrator import ContentIntegrator +from modules.features.aichat.serviceGeneration.subContentIntegrator import ContentIntegrator from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/services/serviceGeneration/subContentIntegrator.py b/modules/features/aichat/serviceGeneration/subContentIntegrator.py similarity index 100% rename from modules/services/serviceGeneration/subContentIntegrator.py rename to modules/features/aichat/serviceGeneration/subContentIntegrator.py diff --git a/modules/services/serviceGeneration/subDocumentUtility.py b/modules/features/aichat/serviceGeneration/subDocumentUtility.py similarity index 100% rename from modules/services/serviceGeneration/subDocumentUtility.py rename to modules/features/aichat/serviceGeneration/subDocumentUtility.py diff --git a/modules/services/serviceGeneration/subJsonSchema.py b/modules/features/aichat/serviceGeneration/subJsonSchema.py similarity index 100% rename from modules/services/serviceGeneration/subJsonSchema.py rename to modules/features/aichat/serviceGeneration/subJsonSchema.py diff --git a/modules/services/serviceGeneration/subPromptBuilderGeneration.py b/modules/features/aichat/serviceGeneration/subPromptBuilderGeneration.py similarity index 100% rename from modules/services/serviceGeneration/subPromptBuilderGeneration.py rename to modules/features/aichat/serviceGeneration/subPromptBuilderGeneration.py diff --git a/modules/services/serviceGeneration/subStructureGenerator.py b/modules/features/aichat/serviceGeneration/subStructureGenerator.py similarity index 100% rename from modules/services/serviceGeneration/subStructureGenerator.py rename to modules/features/aichat/serviceGeneration/subStructureGenerator.py diff --git a/modules/services/serviceWeb/mainServiceWeb.py b/modules/features/aichat/serviceWeb/mainServiceWeb.py similarity index 100% rename from modules/services/serviceWeb/mainServiceWeb.py rename to modules/features/aichat/serviceWeb/mainServiceWeb.py diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py new file mode 100644 index 00000000..a0b8ba0f --- /dev/null +++ b/modules/features/automation/mainAutomation.py @@ -0,0 +1,148 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Automation Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "automation" +FEATURE_LABEL = {"en": "Automation", "de": "Automatisierung", "fr": "Automatisation"} +FEATURE_ICON = "mdi-cog-clockwise" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.automation.definitions", + "label": {"en": "Automation Definitions", "de": "Automatisierungs-Definitionen", "fr": "Définitions d'automatisation"}, + "meta": {"area": "definitions"} + }, + { + "objectKey": "ui.feature.automation.templates", + "label": {"en": "Templates", "de": "Vorlagen", "fr": "Modèles"}, + "meta": {"area": "templates"} + }, + { + "objectKey": "ui.feature.automation.logs", + "label": {"en": "Execution Logs", "de": "Ausführungsprotokolle", "fr": "Journaux d'exécution"}, + "meta": {"area": "logs"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.automation.create", + "label": {"en": "Create Automation", "de": "Automatisierung erstellen", "fr": "Créer automatisation"}, + "meta": {"endpoint": "/api/automations", "method": "POST"} + }, + { + "objectKey": "resource.feature.automation.update", + "label": {"en": "Update Automation", "de": "Automatisierung aktualisieren", "fr": "Modifier automatisation"}, + "meta": {"endpoint": "/api/automations/{automationId}", "method": "PUT"} + }, + { + "objectKey": "resource.feature.automation.delete", + "label": {"en": "Delete Automation", "de": "Automatisierung löschen", "fr": "Supprimer automatisation"}, + "meta": {"endpoint": "/api/automations/{automationId}", "method": "DELETE"} + }, + { + "objectKey": "resource.feature.automation.execute", + "label": {"en": "Execute Automation", "de": "Automatisierung ausführen", "fr": "Exécuter automatisation"}, + "meta": {"endpoint": "/api/automations/{automationId}/execute", "method": "POST"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "automation-admin", + "description": { + "en": "Automation Administrator - Full access to automation configuration and execution", + "de": "Automatisierungs-Administrator - Vollzugriff auf Automatisierungs-Konfiguration und Ausführung", + "fr": "Administrateur automatisation - Accès complet à la configuration et exécution" + } + }, + { + "roleLabel": "automation-editor", + "description": { + "en": "Automation Editor - Create and modify automations", + "de": "Automatisierungs-Editor - Automatisierungen erstellen und bearbeiten", + "fr": "Éditeur automatisation - Créer et modifier les automatisations" + } + }, + { + "roleLabel": "automation-viewer", + "description": { + "en": "Automation Viewer - View automations and execution results", + "de": "Automatisierungs-Betrachter - Automatisierungen und Ausführungsergebnisse einsehen", + "fr": "Visualiseur automatisation - Consulter les automatisations et résultats" + } + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """ + Register this feature's RBAC objects in the catalog. + + Args: + catalogService: The RBAC catalog service instance + + Returns: + True if registration was successful + """ + try: + # Register UI objects + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + # Register Resource objects + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False diff --git a/modules/routes/routeDataAutomation.py b/modules/features/automation/routeFeatureAutomation.py similarity index 96% rename from modules/routes/routeDataAutomation.py rename to modules/features/automation/routeFeatureAutomation.py index db6affab..95c93334 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -13,13 +13,13 @@ import logging import json # Import interfaces and models -from modules.interfaces.interfaceDbChat import getInterface as getChatInterface +from modules.features.aichat.interfaceFeatureAiChat import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions -from modules.features.workflow import executeAutomation -from modules.features.workflow.subAutomationTemplates import getAutomationTemplates +from modules.workflows.automation import executeAutomation +from .subAutomationTemplates import getAutomationTemplates # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/features/workflow/subAutomationTemplates.py b/modules/features/automation/subAutomationTemplates.py similarity index 100% rename from modules/features/workflow/subAutomationTemplates.py rename to modules/features/automation/subAutomationTemplates.py diff --git a/modules/features/workflow/subAutomationUtils.py b/modules/features/automation/subAutomationUtils.py similarity index 100% rename from modules/features/workflow/subAutomationUtils.py rename to modules/features/automation/subAutomationUtils.py diff --git a/modules/datamodels/datamodelChatbot.py b/modules/features/chatbot/datamodelFeatureChatbot.py similarity index 100% rename from modules/datamodels/datamodelChatbot.py rename to modules/features/chatbot/datamodelFeatureChatbot.py diff --git a/modules/interfaces/interfaceDbChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py similarity index 99% rename from modules/interfaces/interfaceDbChatbot.py rename to modules/features/chatbot/interfaceFeatureChatbot.py index 44f124b5..bce7d43e 100644 --- a/modules/interfaces/interfaceDbChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChat import ( +from .datamodelFeatureChatbot import ( ChatDocument, ChatStat, ChatLog, diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 43503339..a82e89dc 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -4,16 +4,120 @@ Simple chatbot feature - basic implementation. User input is processed by AI to create list of needed queries. Those queries get streamed back. + +This module also handles feature initialization and RBAC catalog registration. """ import logging + +# Feature metadata for RBAC catalog +FEATURE_CODE = "chatbot" +FEATURE_LABEL = {"en": "Chatbot", "de": "Chatbot", "fr": "Chatbot"} +FEATURE_ICON = "mdi-robot" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.chatbot.conversations", + "label": {"en": "Conversations", "de": "Konversationen", "fr": "Conversations"}, + "meta": {"area": "conversations"} + }, + { + "objectKey": "ui.feature.chatbot.settings", + "label": {"en": "Settings", "de": "Einstellungen", "fr": "Paramètres"}, + "meta": {"area": "settings"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.chatbot.start", + "label": {"en": "Start Chatbot", "de": "Chatbot starten", "fr": "Démarrer chatbot"}, + "meta": {"endpoint": "/api/chatbot/start/stream", "method": "POST"} + }, + { + "objectKey": "resource.feature.chatbot.stop", + "label": {"en": "Stop Chatbot", "de": "Chatbot stoppen", "fr": "Arrêter chatbot"}, + "meta": {"endpoint": "/api/chatbot/stop/{workflowId}", "method": "POST"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "chatbot-admin", + "description": { + "en": "Chatbot Administrator - Full access to chatbot settings and all conversations", + "de": "Chatbot-Administrator - Vollzugriff auf Chatbot-Einstellungen und alle Konversationen", + "fr": "Administrateur chatbot - Accès complet aux paramètres et conversations" + } + }, + { + "roleLabel": "chatbot-user", + "description": { + "en": "Chatbot User - Use chatbot and view own conversations", + "de": "Chatbot-Benutzer - Chatbot nutzen und eigene Konversationen einsehen", + "fr": "Utilisateur chatbot - Utiliser le chatbot et consulter ses conversations" + } + }, +] + + +def getFeatureDefinition(): + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects(): + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects(): + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles(): + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """Register this feature's RBAC objects in the catalog.""" + try: + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + return True + except Exception as e: + logging.getLogger(__name__).error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False import json import uuid import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -335,7 +439,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChat import ChatLog + from modules.features.aichat.datamodelFeatureAiChat import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/routes/routeFeatureChatbot.py b/modules/features/chatbot/routeFeatureChatbot.py similarity index 98% rename from modules/routes/routeFeatureChatbot.py rename to modules/features/chatbot/routeFeatureChatbot.py index 977158f0..3c97c753 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/features/chatbot/routeFeatureChatbot.py @@ -18,19 +18,19 @@ from modules.shared.timeUtils import parseTimestamp from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChat as interfaceDbChat +from . import interfaceFeatureChatbot as interfaceDbChat from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from .datamodelFeatureChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature -from modules.features.chatbot import chatProcess -from modules.features.chatbot.eventManager import get_event_manager +from . import chatProcess +from .eventManager import get_event_manager # Import workflow control functions -from modules.features.workflow import chatStop +from modules.workflows.automation import chatStop # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/features/dynamicOptions/mainDynamicOptions.py b/modules/features/dynamicOptions/mainDynamicOptions.py deleted file mode 100644 index e8c9a4ff..00000000 --- a/modules/features/dynamicOptions/mainDynamicOptions.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Dynamic Options API feature module. -Provides dynamic options for frontend select/multiselect fields. -""" - -import logging -from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelUam import User - -logger = logging.getLogger(__name__) - -# Standard role definitions (fallback if database is not available) -STANDARD_ROLES = [ - {"value": "sysadmin", "label": {"en": "System Administrator", "fr": "Administrateur système"}}, - {"value": "admin", "label": {"en": "Administrator", "fr": "Administrateur"}}, - {"value": "user", "label": {"en": "User", "fr": "Utilisateur"}}, - {"value": "viewer", "label": {"en": "Viewer", "fr": "Visualiseur"}}, -] - -# Authentication authority options -AUTH_AUTHORITY_OPTIONS = [ - {"value": "local", "label": {"en": "Local", "fr": "Local"}}, - {"value": "google", "label": {"en": "Google", "fr": "Google"}}, - {"value": "msft", "label": {"en": "Microsoft", "fr": "Microsoft"}}, -] - -# Connection status options -# Note: Matches ConnectionStatus enum values (active, expired, revoked, pending) -# Plus "error" for error states (not in enum but used in UI) -CONNECTION_STATUS_OPTIONS = [ - {"value": "active", "label": {"en": "Active", "fr": "Actif"}}, - {"value": "expired", "label": {"en": "Expired", "fr": "Expiré"}}, - {"value": "revoked", "label": {"en": "Revoked", "fr": "Révoqué"}}, - {"value": "pending", "label": {"en": "Pending", "fr": "En attente"}}, - {"value": "error", "label": {"en": "Error", "fr": "Erreur"}}, -] - - -def getOptions(optionsName: str, services, currentUser: Optional[User] = None) -> List[Dict[str, Any]]: - """ - Get options for a given options name. - - Args: - optionsName: Name of the options set to retrieve (e.g., "user.role", "user.connection") - services: Services instance for data access - currentUser: Optional current user for context-aware options - - Returns: - List of option dictionaries with "value" and "label" keys - - Raises: - ValueError: If optionsName is not recognized - """ - logger.debug(f"getOptions called with optionsName='{optionsName}' (repr: {repr(optionsName)})") - optionsNameLower = optionsName.lower() - logger.debug(f"optionsNameLower='{optionsNameLower}'") - - if optionsNameLower == "user.role": - # Fetch roles from database - if currentUser: - try: - roles = services.interfaceDbApp.getAllRoles() - - # Convert Role objects to options format - options = [] - for role in roles: - # Use English description as label, fallback to roleLabel - # Handle TextMultilingual object - if hasattr(role.description, 'get_text'): - # TextMultilingual object - label = role.description.get_text('en') - elif isinstance(role.description, dict): - # Dict format (backward compatibility) - label = role.description.get("en", role.roleLabel) - else: - # Fallback to roleLabel - label = role.roleLabel - - options.append({ - "value": role.roleLabel, - "label": label - }) - - # If no roles in database, return standard roles as fallback - if options: - return options - except Exception as e: - logger.warning(f"Error fetching roles from database, using fallback: {e}") - - # Fallback to standard roles if database fetch fails or no user context - return STANDARD_ROLES - - elif optionsNameLower == "auth.authority": - return AUTH_AUTHORITY_OPTIONS - - elif optionsNameLower == "connection.status": - return CONNECTION_STATUS_OPTIONS - - elif optionsNameLower == "user.connection": - # Dynamic options: Get user connections from database - if not currentUser: - return [] - - try: - connections = services.interfaceDbApp.getUserConnections(currentUser.id) - - return [ - { - "value": conn.id, - "label": { - "en": f"{conn.authority.value} - {conn.externalUsername or conn.externalId}", - "fr": f"{conn.authority.value} - {conn.externalUsername or conn.externalId}" - } - } - for conn in connections - ] - except Exception as e: - logger.error(f"Error fetching user connections for options: {e}") - return [] - - elif optionsNameLower in ("user", "users"): - # Dynamic options: Get all users for the current mandate - if not currentUser: - return [] - - try: - users = services.interfaceDbApp.getUsersByMandate(services.mandateId) - - # Handle both list and PaginatedResult - if hasattr(users, 'items'): - userList = users.items - else: - userList = users - - return [ - { - "value": user.id, - "label": user.fullName or user.username or user.email or user.id - } - for user in userList - ] - except Exception as e: - logger.error(f"Error fetching users for options: {e}") - return [] - - elif optionsNameLower in ("trusteeorganisation", "trustee.organisation"): - # Dynamic options: Get all trustee organisations - if not currentUser: - return [] - - try: - result = services.interfaceDbTrustee.getAllOrganisations() - - # Handle PaginatedResult - items = result.items if hasattr(result, 'items') else result - - return [ - { - "value": org.get("id") if isinstance(org, dict) else org.id, - "label": org.get("label") if isinstance(org, dict) else org.label - } - for org in items - ] - except Exception as e: - logger.error(f"Error fetching trustee organisations for options: {e}") - return [] - - elif optionsNameLower in ("trusteerole", "trustee.role"): - # Dynamic options: Get all trustee roles - if not currentUser: - return [] - - try: - result = services.interfaceDbTrustee.getAllRoles() - - # Handle PaginatedResult - items = result.items if hasattr(result, 'items') else result - - return [ - { - "value": role.get("id") if isinstance(role, dict) else role.id, - # TrusteeRole uses 'desc' field, not 'label' - "label": role.get("desc", role.get("id")) if isinstance(role, dict) else getattr(role, "desc", role.id) - } - for role in items - ] - except Exception as e: - logger.error(f"Error fetching trustee roles for options: {e}") - return [] - - elif optionsNameLower in ("trusteecontract", "trustee.contract"): - # Dynamic options: Get all trustee contracts - if not currentUser: - return [] - - try: - result = services.interfaceDbTrustee.getAllContracts() - - # Handle PaginatedResult - items = result.items if hasattr(result, 'items') else result - - return [ - { - "value": contract.get("id") if isinstance(contract, dict) else contract.id, - "label": contract.get("label") if isinstance(contract, dict) else (contract.get("name") if isinstance(contract, dict) else getattr(contract, "label", getattr(contract, "name", contract.id))) - } - for contract in items - ] - except Exception as e: - logger.error(f"Error fetching trustee contracts for options: {e}") - return [] - - else: - logger.error(f"Unknown options name: '{optionsName}' (lower: '{optionsNameLower}')") - raise ValueError(f"Unknown options name: {optionsName}") - - -def getAvailableOptionsNames() -> List[str]: - """ - Get list of all available options names. - - Returns: - List of available options names - """ - return [ - "user.role", - "auth.authority", - "connection.status", - "user.connection", - "User", - "TrusteeOrganisation", - "TrusteeRole", - "TrusteeContract", - ] - diff --git a/modules/features/featureRegistry.py b/modules/features/featureRegistry.py new file mode 100644 index 00000000..29eb35c9 --- /dev/null +++ b/modules/features/featureRegistry.py @@ -0,0 +1,117 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Feature Registry for Plug&Play Feature Container Loading. +Dynamically discovers and loads feature containers from the features directory. +""" + +import os +import glob +import importlib +import logging +from typing import List, Dict, Any +from fastapi import FastAPI + +logger = logging.getLogger(__name__) + +# Path to the features directory +FEATURES_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def discoverFeatureContainers() -> List[str]: + """ + Discover all feature container directories by filename pattern. + A valid feature container has a routeFeature*.py file. + """ + containers = [] + pattern = os.path.join(FEATURES_DIR, "*", "routeFeature*.py") + + for filepath in glob.glob(pattern): + featureDir = os.path.basename(os.path.dirname(filepath)) + if featureDir not in containers and not featureDir.startswith("_"): + containers.append(featureDir) + + return sorted(containers) + + +def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: + """ + Dynamically load and register routers from all discovered feature containers. + """ + results = {} + pattern = os.path.join(FEATURES_DIR, "*", "routeFeature*.py") + + for filepath in glob.glob(pattern): + featureDir = os.path.basename(os.path.dirname(filepath)) + routerFile = os.path.basename(filepath)[:-3] # Remove .py + + if featureDir.startswith("_"): + continue + + try: + modulePath = f"modules.features.{featureDir}.{routerFile}" + module = importlib.import_module(modulePath) + + if hasattr(module, "router"): + app.include_router(module.router) + logger.info(f"Loaded router: {featureDir}") + results[featureDir] = {"status": "loaded", "module": modulePath} + else: + logger.warning(f"No 'router' in {modulePath}") + results[featureDir] = {"status": "no_router_object"} + + except Exception as e: + logger.error(f"Failed to load router from {featureDir}: {e}") + results[featureDir] = {"status": "error", "error": str(e)} + + return results + + +def loadFeatureMainModules() -> Dict[str, Any]: + """ + Dynamically load main modules from all discovered feature containers. + """ + mainModules = {} + pattern = os.path.join(FEATURES_DIR, "*", "main*.py") + + for filepath in glob.glob(pattern): + filename = os.path.basename(filepath) + if filename == "__init__.py": + continue + + featureDir = os.path.basename(os.path.dirname(filepath)) + if featureDir.startswith("_"): + continue + + mainFile = filename[:-3] # Remove .py + + try: + modulePath = f"modules.features.{featureDir}.{mainFile}" + module = importlib.import_module(modulePath) + mainModules[featureDir] = module + logger.debug(f"Loaded main module: {featureDir}") + except Exception as e: + logger.error(f"Failed to load main module from {featureDir}: {e}") + + return mainModules + + +def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: + """ + Register all features' RBAC objects in the catalog. + """ + mainModules = loadFeatureMainModules() + results = {} + + for featureName, module in mainModules.items(): + if hasattr(module, "registerFeature"): + try: + success = module.registerFeature(catalogService) + results[featureName] = success + if success: + logger.info(f"Registered RBAC objects: {featureName}") + except Exception as e: + logger.error(f"Error registering {featureName}: {e}") + results[featureName] = False + + return results diff --git a/modules/features/featuresLifecycle.py b/modules/features/featuresLifecycle.py deleted file mode 100644 index 0e747676..00000000 --- a/modules/features/featuresLifecycle.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -import logging -from modules.services import getInterface as getServices - -logger = logging.getLogger(__name__) - -async def start(eventUser) -> None: - """ Start feature triggers and background managers - - Args: - eventUser: System-level event user for background operations (provided by app.py) - """ - - # Feature Workflow (Automation) - if eventUser: - try: - from modules.features.workflow import syncAutomationEvents - from modules.shared.callbackRegistry import callbackRegistry - - # Get services for event user (provides access to interfaces) - services = getServices(eventUser, None) - - # Register callback for automation changes - async def onAutomationChanged(chatInterface): - """Callback triggered when automations are created/updated/deleted.""" - # Get services for event user to pass to syncAutomationEvents - eventServices = getServices(eventUser, None) - await syncAutomationEvents(eventServices, eventUser) - - callbackRegistry.register('automation.changed', onAutomationChanged) - logger.info("Workflow: Registered change callback") - - # Initial sync on startup - use services - await syncAutomationEvents(services, eventUser) - logger.info("Workflow: Events synced on startup") - except Exception as e: - logger.error(f"Workflow: Error setting up events on startup: {str(e)}") - # Don't fail startup if automation sync fails - - - # Feature ... - - return True - - - -async def stop(eventUser) -> None: - """ Stop feature triggers and background managers - - Args: - eventUser: System-level event user (provided by app.py) - """ - - # Feature Workflow (Automation) - # Callbacks will remain registered (acceptable for shutdown) - logger.info("Workflow: Callbacks remain registered (will be cleaned up on shutdown)") - - - # Feature ... - - return True diff --git a/modules/datamodels/datamodelNeutralizer.py b/modules/features/neutralizer/datamodelFeatureNeutralizer.py similarity index 100% rename from modules/datamodels/datamodelNeutralizer.py rename to modules/features/neutralizer/datamodelFeatureNeutralizer.py diff --git a/modules/features/neutralizePlayground/mainNeutralizePlayground.py b/modules/features/neutralizer/mainNeutralizePlayground.py similarity index 99% rename from modules/features/neutralizePlayground/mainNeutralizePlayground.py rename to modules/features/neutralizer/mainNeutralizePlayground.py index d10dc8ec..c5932117 100644 --- a/modules/features/neutralizePlayground/mainNeutralizePlayground.py +++ b/modules/features/neutralizer/mainNeutralizePlayground.py @@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlparse, unquote from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig +from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig from modules.services import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/features/neutralizer/mainNeutralizer.py b/modules/features/neutralizer/mainNeutralizer.py new file mode 100644 index 00000000..44d495c4 --- /dev/null +++ b/modules/features/neutralizer/mainNeutralizer.py @@ -0,0 +1,125 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Neutralizer Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "neutralization" +FEATURE_LABEL = {"en": "Neutralization", "de": "Neutralisierung", "fr": "Neutralisation"} +FEATURE_ICON = "mdi-shield-check" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.neutralizer.playground", + "label": {"en": "Playground", "de": "Spielwiese", "fr": "Bac à sable"}, + "meta": {"area": "playground"} + }, + { + "objectKey": "ui.feature.neutralizer.config", + "label": {"en": "Configuration", "de": "Konfiguration", "fr": "Configuration"}, + "meta": {"area": "config"} + }, + { + "objectKey": "ui.feature.neutralizer.attributes", + "label": {"en": "Attributes", "de": "Attribute", "fr": "Attributs"}, + "meta": {"area": "attributes"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.neutralizer.process.text", + "label": {"en": "Process Text", "de": "Text verarbeiten", "fr": "Traiter texte"}, + "meta": {"endpoint": "/api/neutralization/process/text", "method": "POST"} + }, + { + "objectKey": "resource.feature.neutralizer.process.files", + "label": {"en": "Process Files", "de": "Dateien verarbeiten", "fr": "Traiter fichiers"}, + "meta": {"endpoint": "/api/neutralization/process/files", "method": "POST"} + }, + { + "objectKey": "resource.feature.neutralizer.config.update", + "label": {"en": "Update Config", "de": "Konfiguration aktualisieren", "fr": "Mettre à jour config"}, + "meta": {"endpoint": "/api/neutralization/config", "method": "PUT"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "neutralization-admin", + "description": { + "en": "Neutralization Administrator - Full access to neutralization settings and data", + "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", + "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" + } + }, + { + "roleLabel": "neutralization-analyst", + "description": { + "en": "Neutralization Analyst - Analyze and process neutralization data", + "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", + "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" + } + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """Register this feature's RBAC objects in the catalog.""" + try: + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False diff --git a/modules/routes/routeFeatureNeutralization.py b/modules/features/neutralizer/routeFeatureNeutralizer.py similarity index 97% rename from modules/routes/routeFeatureNeutralization.py rename to modules/features/neutralizer/routeFeatureNeutralizer.py index 04d034dc..be262e47 100644 --- a/modules/routes/routeFeatureNeutralization.py +++ b/modules/features/neutralizer/routeFeatureNeutralizer.py @@ -8,8 +8,8 @@ import logging from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes -from modules.features.neutralizePlayground.mainNeutralizePlayground import NeutralizationPlayground +from .datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes +from .mainNeutralizePlayground import NeutralizationPlayground # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/services/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py similarity index 95% rename from modules/services/serviceNeutralization/mainServiceNeutralization.py rename to modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py index 3d2198b6..fb47c188 100644 --- a/modules/services/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py @@ -13,14 +13,14 @@ import re import json from typing import Dict, List, Any, Optional -from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes +from modules.features.neutralizer.datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes # Import all necessary classes and functions for neutralization -from modules.services.serviceNeutralization.subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute -from modules.services.serviceNeutralization.subProcessText import TextProcessor, PlainText -from modules.services.serviceNeutralization.subProcessList import ListProcessor, TableData -from modules.services.serviceNeutralization.subProcessBinary import BinaryProcessor -from modules.services.serviceNeutralization.subPatterns import HeaderPatterns, DataPatterns, TextTablePatterns +from .subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute +from .subProcessText import TextProcessor, PlainText +from .subProcessList import ListProcessor, TableData +from .subProcessBinary import BinaryProcessor +from .subPatterns import HeaderPatterns, DataPatterns, TextTablePatterns logger = logging.getLogger(__name__) diff --git a/modules/services/serviceNeutralization/subParseString.py b/modules/features/neutralizer/serviceNeutralization/subParseString.py similarity index 98% rename from modules/services/serviceNeutralization/subParseString.py rename to modules/features/neutralizer/serviceNeutralization/subParseString.py index 160719c9..1a1b54ad 100644 --- a/modules/services/serviceNeutralization/subParseString.py +++ b/modules/features/neutralizer/serviceNeutralization/subParseString.py @@ -8,7 +8,7 @@ Handles pattern matching and replacement for emails, phones, addresses, IDs and import re import uuid from typing import Dict, List, Tuple, Any -from modules.services.serviceNeutralization.subPatterns import DataPatterns, findPatternsInText +from .subPatterns import DataPatterns, findPatternsInText class StringParser: """Handles string parsing and replacement operations""" diff --git a/modules/services/serviceNeutralization/subPatterns.py b/modules/features/neutralizer/serviceNeutralization/subPatterns.py similarity index 100% rename from modules/services/serviceNeutralization/subPatterns.py rename to modules/features/neutralizer/serviceNeutralization/subPatterns.py diff --git a/modules/services/serviceNeutralization/subProcessBinary.py b/modules/features/neutralizer/serviceNeutralization/subProcessBinary.py similarity index 100% rename from modules/services/serviceNeutralization/subProcessBinary.py rename to modules/features/neutralizer/serviceNeutralization/subProcessBinary.py diff --git a/modules/services/serviceNeutralization/subProcessCommon.py b/modules/features/neutralizer/serviceNeutralization/subProcessCommon.py similarity index 100% rename from modules/services/serviceNeutralization/subProcessCommon.py rename to modules/features/neutralizer/serviceNeutralization/subProcessCommon.py diff --git a/modules/services/serviceNeutralization/subProcessList.py b/modules/features/neutralizer/serviceNeutralization/subProcessList.py similarity index 96% rename from modules/services/serviceNeutralization/subProcessList.py rename to modules/features/neutralizer/serviceNeutralization/subProcessList.py index 3996ccbc..97721535 100644 --- a/modules/services/serviceNeutralization/subProcessList.py +++ b/modules/features/neutralizer/serviceNeutralization/subProcessList.py @@ -11,8 +11,8 @@ import xml.etree.ElementTree as ET from typing import Dict, List, Any, Union from dataclasses import dataclass from io import StringIO -from modules.services.serviceNeutralization.subParseString import StringParser -from modules.services.serviceNeutralization.subPatterns import getPatternForHeader, HeaderPatterns +from .subParseString import StringParser +from .subPatterns import getPatternForHeader, HeaderPatterns @dataclass class TableData: @@ -158,7 +158,7 @@ class ListProcessor: processedAttrs[attrName] = self.string_parser.mapping[attrValue] else: # Check if attribute value matches any data patterns - from modules.services.serviceNeutralization.subPatterns import findPatternsInText, DataPatterns + from .subPatterns import findPatternsInText, DataPatterns matches = findPatternsInText(attrValue, DataPatterns.patterns) if matches: patternName = matches[0][0] @@ -193,7 +193,7 @@ class ListProcessor: # Skip if already a placeholder if not self.string_parser._isPlaceholder(text): # Check if text matches any patterns - from modules.services.serviceNeutralization.subPatterns import findPatternsInText, DataPatterns + from .subPatterns import findPatternsInText, DataPatterns patternMatches = findPatternsInText(text, DataPatterns.patterns) if patternMatches: diff --git a/modules/services/serviceNeutralization/subProcessText.py b/modules/features/neutralizer/serviceNeutralization/subProcessText.py similarity index 97% rename from modules/services/serviceNeutralization/subProcessText.py rename to modules/features/neutralizer/serviceNeutralization/subProcessText.py index 5366fc1b..eea270b9 100644 --- a/modules/services/serviceNeutralization/subProcessText.py +++ b/modules/features/neutralizer/serviceNeutralization/subProcessText.py @@ -7,7 +7,7 @@ Handles plain text processing without header information from typing import Dict, List, Any from dataclasses import dataclass -from modules.services.serviceNeutralization.subParseString import StringParser +from .subParseString import StringParser @dataclass class PlainText: diff --git a/modules/datamodels/datamodelRealEstate.py b/modules/features/realEstate/datamodelFeatureRealEstate.py similarity index 100% rename from modules/datamodels/datamodelRealEstate.py rename to modules/features/realEstate/datamodelFeatureRealEstate.py diff --git a/modules/interfaces/interfaceDbRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py similarity index 99% rename from modules/interfaces/interfaceDbRealEstate.py rename to modules/features/realEstate/interfaceFeatureRealEstate.py index 179ec6dd..545be2c0 100644 --- a/modules/interfaces/interfaceDbRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -6,7 +6,7 @@ Handles CRUD operations on Real Estate entities (Projekt, Parzelle, etc.). import logging from typing import Dict, Any, List, Optional, Union -from modules.datamodels.datamodelRealEstate import ( +from .datamodelFeatureRealEstate import ( Projekt, Parzelle, Dokument, diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index bc729e0d..76f658ba 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -2,16 +2,128 @@ Real Estate feature main logic. Handles database operations with AI-powered natural language processing. Stateless implementation without session management. + +This module also handles feature initialization and RBAC catalog registration. """ import logging + +# Feature metadata for RBAC catalog +FEATURE_CODE = "realestate" +FEATURE_LABEL = {"en": "Real Estate", "de": "Immobilien", "fr": "Immobilier"} +FEATURE_ICON = "mdi-home-city" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.realestate.projects", + "label": {"en": "Projects", "de": "Projekte", "fr": "Projets"}, + "meta": {"area": "projects"} + }, + { + "objectKey": "ui.feature.realestate.parcels", + "label": {"en": "Parcels", "de": "Parzellen", "fr": "Parcelles"}, + "meta": {"area": "parcels"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.realestate.project.create", + "label": {"en": "Create Project", "de": "Projekt erstellen", "fr": "Créer projet"}, + "meta": {"endpoint": "/api/realestate/project", "method": "POST"} + }, + { + "objectKey": "resource.feature.realestate.project.delete", + "label": {"en": "Delete Project", "de": "Projekt löschen", "fr": "Supprimer projet"}, + "meta": {"endpoint": "/api/realestate/project/{projectId}", "method": "DELETE"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "realestate-admin", + "description": { + "en": "Real Estate Administrator - Full access to all property data and settings", + "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", + "fr": "Administrateur immobilier - Accès complet aux données et paramètres" + } + }, + { + "roleLabel": "realestate-manager", + "description": { + "en": "Real Estate Manager - Manage properties and tenants", + "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", + "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" + } + }, + { + "roleLabel": "realestate-viewer", + "description": { + "en": "Real Estate Viewer - View property information", + "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", + "fr": "Visualiseur immobilier - Consulter les informations immobilières" + } + }, +] + + +def getFeatureDefinition(): + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects(): + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects(): + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles(): + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """Register this feature's RBAC objects in the catalog.""" + try: + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + return True + except Exception as e: + logging.getLogger(__name__).error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False import json from typing import Optional, Dict, Any, List from fastapi import HTTPException, status from shapely.geometry import Polygon from shapely.ops import unary_union from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelRealEstate import ( +from .datamodelFeatureRealEstate import ( Projekt, Parzelle, StatusProzess, @@ -23,7 +135,7 @@ from modules.datamodels.datamodelRealEstate import ( Land, ) from modules.services import getInterface as getServices -from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector logger = logging.getLogger(__name__) @@ -905,7 +1017,7 @@ async def executeIntentBasedOperation( elif entity == "Parzelle": # Create Parzelle from parameters # Import Kontext for kontextInformationen - from modules.datamodels.datamodelRealEstate import Kontext, GeoPolylinie + from modules.features.realestate.datamodelFeatureRealEstate import Kontext, GeoPolylinie # Build parzelle data with all extracted parameters parzelle_data = { @@ -990,7 +1102,7 @@ async def executeIntentBasedOperation( } elif entity == "Gemeinde": # Create Gemeinde from parameters - from modules.datamodels.datamodelRealEstate import Gemeinde + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde gemeinde = Gemeinde( mandateId=mandateId, label=parameters.get("label", ""), @@ -1005,7 +1117,7 @@ async def executeIntentBasedOperation( } elif entity == "Kanton": # Create Kanton from parameters - from modules.datamodels.datamodelRealEstate import Kanton + from modules.features.realestate.datamodelFeatureRealEstate import Kanton kanton = Kanton( mandateId=mandateId, label=parameters.get("label", ""), @@ -1020,7 +1132,7 @@ async def executeIntentBasedOperation( } elif entity == "Land": # Create Land from parameters - from modules.datamodels.datamodelRealEstate import Land + from modules.features.realestate.datamodelFeatureRealEstate import Land land = Land( mandateId=mandateId, label=parameters.get("label", ""), @@ -1034,7 +1146,7 @@ async def executeIntentBasedOperation( } elif entity == "Dokument": # Create Dokument from parameters - from modules.datamodels.datamodelRealEstate import Dokument + from modules.features.realestate.datamodelFeatureRealEstate import Dokument dokument = Dokument( mandateId=mandateId, label=parameters.get("label", ""), @@ -1195,7 +1307,7 @@ async def executeIntentBasedOperation( "count": len(parzellen) } elif entity == "Gemeinde": - from modules.datamodels.datamodelRealEstate import Gemeinde + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde gemeindeId = parameters.get("id") if gemeindeId: gemeinde = realEstateInterface.getGemeinde(gemeindeId) @@ -1216,7 +1328,7 @@ async def executeIntentBasedOperation( "count": len(gemeinden) } elif entity == "Kanton": - from modules.datamodels.datamodelRealEstate import Kanton + from modules.features.realestate.datamodelFeatureRealEstate import Kanton kantonId = parameters.get("id") if kantonId: kanton = realEstateInterface.getKanton(kantonId) @@ -1237,7 +1349,7 @@ async def executeIntentBasedOperation( "count": len(kantone) } elif entity == "Land": - from modules.datamodels.datamodelRealEstate import Land + from modules.features.realestate.datamodelFeatureRealEstate import Land landId = parameters.get("id") if landId: land = realEstateInterface.getLand(landId) @@ -1258,7 +1370,7 @@ async def executeIntentBasedOperation( "count": len(laender) } elif entity == "Dokument": - from modules.datamodels.datamodelRealEstate import Dokument + from modules.features.realestate.datamodelFeatureRealEstate import Dokument dokumentId = parameters.get("id") if dokumentId: dokument = realEstateInterface.getDokument(dokumentId) @@ -1322,7 +1434,7 @@ async def executeIntentBasedOperation( "result": updated.model_dump() } elif entity == "Gemeinde": - from modules.datamodels.datamodelRealEstate import Gemeinde + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde gemeindeId = parameters.get("id") if not gemeindeId: raise ValueError("UPDATE operation requires entity ID") @@ -1339,7 +1451,7 @@ async def executeIntentBasedOperation( "result": updated.model_dump() } elif entity == "Kanton": - from modules.datamodels.datamodelRealEstate import Kanton + from modules.features.realestate.datamodelFeatureRealEstate import Kanton kantonId = parameters.get("id") if not kantonId: raise ValueError("UPDATE operation requires entity ID") @@ -1356,7 +1468,7 @@ async def executeIntentBasedOperation( "result": updated.model_dump() } elif entity == "Land": - from modules.datamodels.datamodelRealEstate import Land + from modules.features.realestate.datamodelFeatureRealEstate import Land landId = parameters.get("id") if not landId: raise ValueError("UPDATE operation requires entity ID") @@ -1373,7 +1485,7 @@ async def executeIntentBasedOperation( "result": updated.model_dump() } elif entity == "Dokument": - from modules.datamodels.datamodelRealEstate import Dokument + from modules.features.realestate.datamodelFeatureRealEstate import Dokument dokumentId = parameters.get("id") if not dokumentId: raise ValueError("UPDATE operation requires entity ID") @@ -1419,7 +1531,7 @@ async def executeIntentBasedOperation( "success": success } elif entity == "Gemeinde": - from modules.datamodels.datamodelRealEstate import Gemeinde + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde gemeindeId = parameters.get("id") if not gemeindeId: raise ValueError("DELETE operation requires entity ID") @@ -1431,7 +1543,7 @@ async def executeIntentBasedOperation( "success": success } elif entity == "Kanton": - from modules.datamodels.datamodelRealEstate import Kanton + from modules.features.realestate.datamodelFeatureRealEstate import Kanton kantonId = parameters.get("id") if not kantonId: raise ValueError("DELETE operation requires entity ID") @@ -1443,7 +1555,7 @@ async def executeIntentBasedOperation( "success": success } elif entity == "Land": - from modules.datamodels.datamodelRealEstate import Land + from modules.features.realestate.datamodelFeatureRealEstate import Land landId = parameters.get("id") if not landId: raise ValueError("DELETE operation requires entity ID") @@ -1455,7 +1567,7 @@ async def executeIntentBasedOperation( "success": success } elif entity == "Dokument": - from modules.datamodels.datamodelRealEstate import Dokument + from modules.features.realestate.datamodelFeatureRealEstate import Dokument dokumentId = parameters.get("id") if not dokumentId: raise ValueError("DELETE operation requires entity ID") diff --git a/modules/routes/routeFeatureRealEstate.py b/modules/features/realEstate/routeFeatureRealEstate.py similarity index 99% rename from modules/routes/routeFeatureRealEstate.py rename to modules/features/realEstate/routeFeatureRealEstate.py index 73364345..3cb31a2c 100644 --- a/modules/routes/routeFeatureRealEstate.py +++ b/modules/features/realEstate/routeFeatureRealEstate.py @@ -14,7 +14,7 @@ from modules.auth import limiter, getRequestContext, RequestContext # Import models from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata -from modules.datamodels.datamodelRealEstate import ( +from .datamodelFeatureRealEstate import ( Projekt, Parzelle, Dokument, @@ -26,10 +26,10 @@ from modules.datamodels.datamodelRealEstate import ( ) # Import interfaces -from modules.interfaces.interfaceDbRealestate import getInterface as getRealEstateInterface +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands -from modules.features.realEstate.mainRealEstate import ( +from .mainRealEstate import ( processNaturalLanguageCommand, create_project_with_parcel_data, ) diff --git a/modules/datamodels/datamodelTrustee.py b/modules/features/trustee/datamodelFeatureTrustee.py similarity index 100% rename from modules/datamodels/datamodelTrustee.py rename to modules/features/trustee/datamodelFeatureTrustee.py diff --git a/modules/interfaces/interfaceDbTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py similarity index 99% rename from modules/interfaces/interfaceDbTrustee.py rename to modules/features/trustee/interfaceFeatureTrustee.py index 56ce19b3..dbd8c8aa 100644 --- a/modules/interfaces/interfaceDbTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -16,7 +16,7 @@ from modules.interfaces.interfaceRbac import getRecordsetWithRBAC from modules.security.rbac import RbacClass from modules.datamodels.datamodelUam import User, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext -from modules.datamodels.datamodelTrustee import ( +from .datamodelFeatureTrustee import ( TrusteeOrganisation, TrusteeRole, TrusteeAccess, diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py new file mode 100644 index 00000000..176317a7 --- /dev/null +++ b/modules/features/trustee/mainTrustee.py @@ -0,0 +1,193 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Trustee Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "trustee" +FEATURE_LABEL = {"en": "Trustee", "de": "Treuhand", "fr": "Fiduciaire"} +FEATURE_ICON = "mdi-briefcase" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.trustee.organisations", + "label": {"en": "Organisations", "de": "Organisationen", "fr": "Organisations"}, + "meta": {"area": "organisations"} + }, + { + "objectKey": "ui.feature.trustee.contracts", + "label": {"en": "Contracts", "de": "Verträge", "fr": "Contrats"}, + "meta": {"area": "contracts"} + }, + { + "objectKey": "ui.feature.trustee.contracts.tab.documents", + "label": {"en": "Contract Documents", "de": "Vertragsdokumente", "fr": "Documents contractuels"}, + "meta": {"area": "contracts", "element": "tab.documents"} + }, + { + "objectKey": "ui.feature.trustee.contracts.tab.positions", + "label": {"en": "Contract Positions", "de": "Vertragspositionen", "fr": "Positions contractuelles"}, + "meta": {"area": "contracts", "element": "tab.positions"} + }, + { + "objectKey": "ui.feature.trustee.access", + "label": {"en": "Access Management", "de": "Zugriffsverwaltung", "fr": "Gestion des accès"}, + "meta": {"area": "access"} + }, + { + "objectKey": "ui.feature.trustee.roles", + "label": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, + "meta": {"area": "roles"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.trustee.organisations.create", + "label": {"en": "Create Organisation", "de": "Organisation erstellen", "fr": "Créer organisation"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/organisations", "method": "POST"} + }, + { + "objectKey": "resource.feature.trustee.organisations.update", + "label": {"en": "Update Organisation", "de": "Organisation aktualisieren", "fr": "Modifier organisation"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/organisations/{orgId}", "method": "PUT"} + }, + { + "objectKey": "resource.feature.trustee.organisations.delete", + "label": {"en": "Delete Organisation", "de": "Organisation löschen", "fr": "Supprimer organisation"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/organisations/{orgId}", "method": "DELETE"} + }, + { + "objectKey": "resource.feature.trustee.contracts.create", + "label": {"en": "Create Contract", "de": "Vertrag erstellen", "fr": "Créer contrat"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/contracts", "method": "POST"} + }, + { + "objectKey": "resource.feature.trustee.contracts.update", + "label": {"en": "Update Contract", "de": "Vertrag aktualisieren", "fr": "Modifier contrat"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/contracts/{contractId}", "method": "PUT"} + }, + { + "objectKey": "resource.feature.trustee.contracts.delete", + "label": {"en": "Delete Contract", "de": "Vertrag löschen", "fr": "Supprimer contrat"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/contracts/{contractId}", "method": "DELETE"} + }, + { + "objectKey": "resource.feature.trustee.documents.create", + "label": {"en": "Upload Document", "de": "Dokument hochladen", "fr": "Télécharger document"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/documents", "method": "POST"} + }, + { + "objectKey": "resource.feature.trustee.documents.delete", + "label": {"en": "Delete Document", "de": "Dokument löschen", "fr": "Supprimer document"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/documents/{documentId}", "method": "DELETE"} + }, + { + "objectKey": "resource.feature.trustee.positions.create", + "label": {"en": "Create Position", "de": "Position erstellen", "fr": "Créer position"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/positions", "method": "POST"} + }, + { + "objectKey": "resource.feature.trustee.positions.delete", + "label": {"en": "Delete Position", "de": "Position löschen", "fr": "Supprimer position"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/positions/{positionId}", "method": "DELETE"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "trustee-admin", + "description": { + "en": "Trustee Administrator - Full access to all trustee data and settings", + "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", + "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" + } + }, + { + "roleLabel": "trustee-accountant", + "description": { + "en": "Trustee Accountant - Manage accounting and financial data", + "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", + "fr": "Comptable fiduciaire - Gérer les données comptables et financières" + } + }, + { + "roleLabel": "trustee-client", + "description": { + "en": "Trustee Client - View own accounting data and documents", + "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", + "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" + } + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """ + Register this feature's RBAC objects in the catalog. + + Args: + catalogService: The RBAC catalog service instance + + Returns: + True if registration was successful + """ + try: + # Register UI objects + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + # Register Resource objects + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False diff --git a/modules/routes/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py similarity index 99% rename from modules/routes/routeFeatureTrustee.py rename to modules/features/trustee/routeFeatureTrustee.py index 14b0a9e8..bee52513 100644 --- a/modules/routes/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -18,10 +18,10 @@ import json import io from modules.auth import limiter, getRequestContext, RequestContext -from modules.interfaces.interfaceDbTrustee import getInterface +from .interfaceFeatureTrustee import getInterface from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface -from modules.datamodels.datamodelTrustee import ( +from .datamodelFeatureTrustee import ( TrusteeOrganisation, TrusteeRole, TrusteeAccess, diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 5c252ff6..e5aa72cc 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -10,8 +10,8 @@ import time logger = logging.getLogger(__name__) -from modules.aicore.aicoreModelRegistry import modelRegistry -from modules.aicore.aicoreModelSelector import modelSelector +from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.features.aichat.aicore.aicoreModelSelector import modelSelector from modules.datamodels.datamodelAi import ( AiModel, AiCallOptions, diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index cbf8295a..4b291537 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -30,8 +30,6 @@ from modules.datamodels.datamodelMembership import ( UserMandate, UserMandateRole, ) -from modules.datamodels.datamodelFeatures import Feature - logger = logging.getLogger(__name__) # Password-Hashing @@ -56,15 +54,9 @@ def initBootstrap(db: DatabaseConnector) -> None: # Initialize roles FIRST (needed for AccessRules) initRoles(db) - # Initialize features (trustee, chatbot, etc.) - initFeatures(db) - # Initialize RBAC rules (uses roleIds from roles) initRbacRules(db) - # Initialize AccessRules for feature-template roles (idempotent - adds missing rules) - _initFeatureTemplateRoleAccessRules(db) - # Initialize admin user adminUserId = initAdminUser(db, mandateId) @@ -247,233 +239,6 @@ def initRoles(db: DatabaseConnector) -> None: logger.info("Roles initialization completed") -def initFeatures(db: DatabaseConnector) -> None: - """ - Initialize standard features if they don't exist. - - Features are global definitions that can be instantiated within mandates. - Each feature has a unique code (e.g., 'trustee', 'chatbot'). - - Args: - db: Database connector instance - """ - logger.info("Initializing features") - - # Standard features available in the system - standardFeatures = [ - Feature( - code="trustee", - label={"en": "Trustee", "de": "Treuhand", "fr": "Fiduciaire"}, - icon="mdi-briefcase" - ), - Feature( - code="chatbot", - label={"en": "Chatbot", "de": "Chatbot", "fr": "Chatbot"}, - icon="mdi-robot" - ), - Feature( - code="chatworkflow", - label={"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"}, - icon="mdi-message-cog" - ), - Feature( - code="neutralization", - label={"en": "Neutralization", "de": "Neutralisierung", "fr": "Neutralisation"}, - icon="mdi-shield-check" - ), - Feature( - code="realestate", - label={"en": "Real Estate", "de": "Immobilien", "fr": "Immobilier"}, - icon="mdi-home-city" - ), - ] - - existingFeatures = db.getRecordset(Feature) - existingCodes = {f.get("code") for f in existingFeatures} - - for feature in standardFeatures: - if feature.code not in existingCodes: - try: - db.recordCreate(Feature, feature.model_dump()) - logger.info(f"Created feature: {feature.code}") - except Exception as e: - logger.warning(f"Error creating feature {feature.code}: {e}") - else: - logger.debug(f"Feature {feature.code} already exists") - - # Initialize feature-specific template roles - _initFeatureTemplateRoles(db) - - logger.info("Features initialization completed") - - -def _initFeatureTemplateRoles(db: DatabaseConnector) -> None: - """ - Initialize feature-specific template roles. - - These are global template roles (mandateId=None, featureInstanceId=None) - that get copied when a new FeatureInstance is created. - - Template roles are NOT system roles (isSystemRole=False) and can be - modified or deleted by administrators. - - Args: - db: Database connector instance - """ - logger.info("Initializing feature-specific template roles") - - # Feature-specific template roles definition - # Each feature has its own set of roles with appropriate descriptions - featureTemplateRoles = { - "trustee": [ - { - "roleLabel": "trustee-admin", - "description": { - "en": "Trustee Administrator - Full access to all trustee data and settings", - "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", - "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" - } - }, - { - "roleLabel": "trustee-accountant", - "description": { - "en": "Trustee Accountant - Manage accounting and financial data", - "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", - "fr": "Comptable fiduciaire - Gérer les données comptables et financières" - } - }, - { - "roleLabel": "trustee-client", - "description": { - "en": "Trustee Client - View own accounting data and documents", - "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", - "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" - } - }, - ], - "chatbot": [ - { - "roleLabel": "chatbot-admin", - "description": { - "en": "Chatbot Administrator - Full access to chatbot settings and all conversations", - "de": "Chatbot-Administrator - Vollzugriff auf Chatbot-Einstellungen und alle Konversationen", - "fr": "Administrateur chatbot - Accès complet aux paramètres et conversations" - } - }, - { - "roleLabel": "chatbot-user", - "description": { - "en": "Chatbot User - Use chatbot and view own conversations", - "de": "Chatbot-Benutzer - Chatbot nutzen und eigene Konversationen einsehen", - "fr": "Utilisateur chatbot - Utiliser le chatbot et consulter ses conversations" - } - }, - ], - "chatworkflow": [ - { - "roleLabel": "workflow-admin", - "description": { - "en": "Workflow Administrator - Full access to workflow configuration and execution", - "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", - "fr": "Administrateur workflow - Accès complet à la configuration et exécution" - } - }, - { - "roleLabel": "workflow-editor", - "description": { - "en": "Workflow Editor - Create and modify workflows", - "de": "Workflow-Editor - Workflows erstellen und bearbeiten", - "fr": "Éditeur workflow - Créer et modifier les workflows" - } - }, - { - "roleLabel": "workflow-viewer", - "description": { - "en": "Workflow Viewer - View workflows and execution results", - "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", - "fr": "Visualiseur workflow - Consulter les workflows et résultats" - } - }, - ], - "neutralization": [ - { - "roleLabel": "neutralization-admin", - "description": { - "en": "Neutralization Administrator - Full access to neutralization settings and data", - "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", - "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" - } - }, - { - "roleLabel": "neutralization-analyst", - "description": { - "en": "Neutralization Analyst - Analyze and process neutralization data", - "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", - "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" - } - }, - ], - "realestate": [ - { - "roleLabel": "realestate-admin", - "description": { - "en": "Real Estate Administrator - Full access to all property data and settings", - "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", - "fr": "Administrateur immobilier - Accès complet aux données et paramètres" - } - }, - { - "roleLabel": "realestate-manager", - "description": { - "en": "Real Estate Manager - Manage properties and tenants", - "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", - "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" - } - }, - { - "roleLabel": "realestate-viewer", - "description": { - "en": "Real Estate Viewer - View property information", - "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", - "fr": "Visualiseur immobilier - Consulter les informations immobilières" - } - }, - ], - } - - # Get existing template roles (mandateId=None, featureCode set) - existingRoles = db.getRecordset(Role, recordFilter={"mandateId": None}) - existingRoleKeys = { - (r.get("featureCode"), r.get("roleLabel")) - for r in existingRoles - if r.get("featureCode") is not None - } - - createdCount = 0 - for featureCode, roles in featureTemplateRoles.items(): - for roleDef in roles: - roleKey = (featureCode, roleDef["roleLabel"]) - if roleKey not in existingRoleKeys: - try: - templateRole = Role( - roleLabel=roleDef["roleLabel"], - description=roleDef["description"], - mandateId=None, # Global template role - featureInstanceId=None, - featureCode=featureCode, - isSystemRole=False # Can be deleted by admins - ) - db.recordCreate(Role, templateRole) - createdCount += 1 - logger.info(f"Created template role: {roleDef['roleLabel']} for feature {featureCode}") - except Exception as e: - logger.warning(f"Error creating template role {roleDef['roleLabel']}: {e}") - else: - logger.debug(f"Template role {roleDef['roleLabel']} for {featureCode} already exists") - - logger.info(f"Feature template roles initialization completed ({createdCount} created)") - - def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: """ Get role ID by label, using cache or database lookup. @@ -501,190 +266,6 @@ def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: return None -def _initFeatureTemplateRoleAccessRules(db: DatabaseConnector) -> None: - """ - Initialize AccessRules for feature-template roles. - This is idempotent - only adds rules that don't exist yet. - - Feature-template roles need explicit AccessRules for their respective tables: - - trustee-admin/accountant/client -> TrusteeOrganisation, TrusteeContract, etc. - - chatbot-admin/user -> ChatSession, etc. - - workflow-admin/editor/viewer -> ChatWorkflow, etc. - - Args: - db: Database connector instance - """ - logger.info("Checking feature-template role AccessRules") - - # Define feature-specific table access - featureTableAccess = { - "trustee": { - "tables": [ - "TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", - "TrusteeContract", "TrusteeDocument", "TrusteePosition", - "TrusteePositionDocument" - ], - "roles": { - "trustee-admin": { - "view": True, - "read": AccessLevel.ALL, - "create": AccessLevel.ALL, - "update": AccessLevel.ALL, - "delete": AccessLevel.ALL - }, - "trustee-accountant": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.GROUP, - "update": AccessLevel.GROUP, - "delete": AccessLevel.NONE - }, - "trustee-client": { - "view": True, - "read": AccessLevel.MY, - "create": AccessLevel.NONE, - "update": AccessLevel.NONE, - "delete": AccessLevel.NONE - } - } - }, - "chatbot": { - "tables": ["ChatSession", "ChatMessage"], - "roles": { - "chatbot-admin": { - "view": True, - "read": AccessLevel.ALL, - "create": AccessLevel.ALL, - "update": AccessLevel.ALL, - "delete": AccessLevel.ALL - }, - "chatbot-user": { - "view": True, - "read": AccessLevel.MY, - "create": AccessLevel.MY, - "update": AccessLevel.MY, - "delete": AccessLevel.MY - } - } - }, - "chatworkflow": { - "tables": ["ChatWorkflow", "AutomationDefinition"], - "roles": { - "workflow-admin": { - "view": True, - "read": AccessLevel.ALL, - "create": AccessLevel.ALL, - "update": AccessLevel.ALL, - "delete": AccessLevel.ALL - }, - "workflow-editor": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.GROUP, - "update": AccessLevel.GROUP, - "delete": AccessLevel.NONE - }, - "workflow-viewer": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.NONE, - "update": AccessLevel.NONE, - "delete": AccessLevel.NONE - } - } - }, - "neutralization": { - "tables": ["DataNeutraliserConfig", "DataNeutralizerAttributes"], - "roles": { - "neutralization-admin": { - "view": True, - "read": AccessLevel.ALL, - "create": AccessLevel.ALL, - "update": AccessLevel.ALL, - "delete": AccessLevel.ALL - }, - "neutralization-analyst": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.NONE, - "update": AccessLevel.NONE, - "delete": AccessLevel.NONE - } - } - }, - "realestate": { - "tables": ["Projekt", "Parzelle", "Dokument", "Gemeinde", "Kanton", "Land"], - "roles": { - "realestate-admin": { - "view": True, - "read": AccessLevel.ALL, - "create": AccessLevel.ALL, - "update": AccessLevel.ALL, - "delete": AccessLevel.ALL - }, - "realestate-manager": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.GROUP, - "update": AccessLevel.GROUP, - "delete": AccessLevel.NONE - }, - "realestate-viewer": { - "view": True, - "read": AccessLevel.GROUP, - "create": AccessLevel.NONE, - "update": AccessLevel.NONE, - "delete": AccessLevel.NONE - } - } - } - } - - createdCount = 0 - - for featureCode, featureConfig in featureTableAccess.items(): - tables = featureConfig["tables"] - roles = featureConfig["roles"] - - for roleLabel, permissions in roles.items(): - roleId = _getRoleId(db, roleLabel) - if not roleId: - continue - - for tableName in tables: - # Check if rule already exists - existingRules = db.getRecordset( - AccessRule, - recordFilter={ - "roleId": roleId, - "context": AccessRuleContext.DATA, - "item": tableName - } - ) - - if existingRules: - continue # Rule already exists - - # Create new rule - rule = AccessRule( - roleId=roleId, - context=AccessRuleContext.DATA, - item=tableName, - view=permissions["view"], - read=permissions["read"], - create=permissions["create"], - update=permissions["update"], - delete=permissions["delete"] - ) - db.recordCreate(AccessRule, rule) - createdCount += 1 - - if createdCount > 0: - logger.info(f"Created {createdCount} feature-template role AccessRules") - else: - logger.debug("All feature-template role AccessRules already exist") - - def initRbacRules(db: DatabaseConnector) -> None: """ Initialize RBAC rules if they don't exist. diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index ed8e1fc4..5b6633cb 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -36,7 +36,7 @@ from modules.datamodels.datamodelRbac import ( ) from modules.datamodels.datamodelUam import AccessLevel from modules.datamodels.datamodelSecurity import Token, AuthEvent, TokenStatus -from modules.datamodels.datamodelNeutralizer import ( +from modules.features.neutralizer.datamodelFeatureNeutralizer import ( DataNeutraliserConfig, DataNeutralizerAttributes, ) diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index c62977aa..9202e448 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -10,8 +10,8 @@ from typing import List, Dict, Any from fastapi import status import logging -# Import interfaces and models -import modules.interfaces.interfaceDbChat as interfaceDbChat +# Import interfaces and models from feature containers +import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User @@ -76,9 +76,9 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + from modules.features.aichat.interfaceFeatureAiChat import getInterface as getChatInterface from modules.interfaces.interfaceDbApp import getRootInterface - from modules.features.workflow import syncAutomationEvents + from modules.workflows.automation import syncAutomationEvents chatInterface = getChatInterface(currentUser) # Get event user for sync operation (routes can import from interfaces) diff --git a/modules/routes/routeDataWorkflows.py b/modules/routes/routeDataWorkflows.py index d3b2d825..b4013de1 100644 --- a/modules/routes/routeDataWorkflows.py +++ b/modules/routes/routeDataWorkflows.py @@ -13,13 +13,13 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon # Import auth modules from modules.auth import limiter, getCurrentUser -# Import interfaces -import modules.interfaces.interfaceDbChat as interfaceDbChat -from modules.interfaces.interfaceDbChat import getInterface +# Import interfaces from feature containers +import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat +from modules.features.aichat.interfaceFeatureAiChat import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC -# Import models -from modules.datamodels.datamodelChat import ( +# Import models from feature containers +from modules.features.aichat.datamodelFeatureAiChat import ( ChatWorkflow, ChatMessage, ChatLog, diff --git a/modules/routes/routeOptions.py b/modules/routes/routeOptions.py deleted file mode 100644 index a2acad76..00000000 --- a/modules/routes/routeOptions.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Options API routes for dynamic frontend options. -Provides endpoints for fetching options for select/multiselect fields. -""" - -from fastapi import APIRouter, HTTPException, Depends, Query, Request -from typing import List, Dict, Any -import logging - -from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelUam import User -from modules.features.dynamicOptions.mainDynamicOptions import getOptions, getAvailableOptionsNames -from modules.services import getInterface as getServices - -# Configure logger -logger = logging.getLogger(__name__) - -router = APIRouter( - prefix="/api/options", - tags=["Options"], - responses={404: {"description": "Not found"}} -) - - -@router.get("/{optionsName}", response_model=List[Dict[str, Any]]) -@limiter.limit("120/minute") -async def getOptionsEndpoint( - request: Request, - optionsName: str, - currentUser: User = Depends(getCurrentUser) -) -> List[Dict[str, Any]]: - """ - Get options for a given options name. - - Path Parameters: - - optionsName: Name of the options set (e.g., "user.role", "user.connection") - - Returns: - - List of option dictionaries with "value" and "label" keys - - Examples: - - GET /api/options/user.role - - GET /api/options/user.connection - - GET /api/options/auth.authority - - GET /api/options/connection.status - """ - try: - logger.debug(f"Options request: {optionsName} for user {currentUser.id}") - services = getServices(currentUser, None) - options = getOptions(optionsName, services, currentUser) - logger.debug(f"Options response: {optionsName} returned {len(options)} items") - return options - except ValueError as e: - logger.error(f"ValueError for options {optionsName}: {str(e)}") - raise HTTPException( - status_code=400, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error getting options for {optionsName}: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Failed to get options: {str(e)}" - ) - - -@router.get("/", response_model=List[str]) -@limiter.limit("30/minute") -async def listAvailableOptions( - request: Request, - currentUser: User = Depends(getCurrentUser) -) -> List[str]: - """ - Get list of all available options names. - - Returns: - - List of available options names - """ - try: - return getAvailableOptionsNames() - except Exception as e: - logger.error(f"Error listing available options: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Failed to list options: {str(e)}" - ) diff --git a/modules/security/rbacCatalog.py b/modules/security/rbacCatalog.py new file mode 100644 index 00000000..f52adf21 --- /dev/null +++ b/modules/security/rbacCatalog.py @@ -0,0 +1,151 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +RBAC Catalog Service. +Central registry for Feature RBAC objects (UI and RESOURCE). + +Feature-Container register their RBAC objects via mainXxx.py at startup. +""" + +import logging +from typing import Dict, List, Any, Optional +from threading import Lock + +logger = logging.getLogger(__name__) + + +class RbacCatalogService: + """ + Central RBAC Catalog for Feature UI and RESOURCE objects. + Singleton service that stores all registered RBAC objects from feature containers. + """ + + _instance = None + _lock = Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + + self._uiObjects: Dict[str, Dict[str, Any]] = {} + self._resourceObjects: Dict[str, Dict[str, Any]] = {} + self._featureDefinitions: Dict[str, Dict[str, Any]] = {} + self._templateRoles: Dict[str, List[Dict[str, Any]]] = {} + self._initialized = True + logger.info("RBAC Catalog Service initialized") + + def registerUiObject(self, featureCode: str, objectKey: str, label: Dict[str, str], meta: Optional[Dict[str, Any]] = None) -> bool: + """Register a UI object for a feature.""" + try: + self._uiObjects[objectKey] = {"objectKey": objectKey, "featureCode": featureCode, "label": label, "meta": meta or {}, "type": "UI"} + return True + except Exception as e: + logger.error(f"Failed to register UI object {objectKey}: {e}") + return False + + def registerResourceObject(self, featureCode: str, objectKey: str, label: Dict[str, str], meta: Optional[Dict[str, Any]] = None) -> bool: + """Register a RESOURCE object for a feature.""" + try: + self._resourceObjects[objectKey] = {"objectKey": objectKey, "featureCode": featureCode, "label": label, "meta": meta or {}, "type": "RESOURCE"} + return True + except Exception as e: + logger.error(f"Failed to register RESOURCE object {objectKey}: {e}") + return False + + def registerFeatureDefinition(self, featureCode: str, label: Dict[str, str], icon: str) -> bool: + """Register a feature definition.""" + try: + self._featureDefinitions[featureCode] = {"code": featureCode, "label": label, "icon": icon} + return True + except Exception as e: + logger.error(f"Failed to register feature definition {featureCode}: {e}") + return False + + def registerTemplateRoles(self, featureCode: str, roles: List[Dict[str, Any]]) -> bool: + """Register template roles for a feature.""" + try: + self._templateRoles[featureCode] = roles + return True + except Exception as e: + logger.error(f"Failed to register template roles for {featureCode}: {e}") + return False + + def getUiObjects(self, featureCode: Optional[str] = None) -> List[Dict[str, Any]]: + """Get all UI objects, optionally filtered by feature.""" + if featureCode: + return [obj for obj in self._uiObjects.values() if obj["featureCode"] == featureCode] + return list(self._uiObjects.values()) + + def getResourceObjects(self, featureCode: Optional[str] = None) -> List[Dict[str, Any]]: + """Get all RESOURCE objects, optionally filtered by feature.""" + if featureCode: + return [obj for obj in self._resourceObjects.values() if obj["featureCode"] == featureCode] + return list(self._resourceObjects.values()) + + def getAllObjects(self, featureCode: Optional[str] = None) -> List[Dict[str, Any]]: + """Get all RBAC objects (UI + RESOURCE), optionally filtered by feature.""" + return self.getUiObjects(featureCode) + self.getResourceObjects(featureCode) + + def getFeatureDefinitions(self) -> List[Dict[str, Any]]: + """Get all registered feature definitions.""" + return list(self._featureDefinitions.values()) + + def getFeatureDefinition(self, featureCode: str) -> Optional[Dict[str, Any]]: + """Get a specific feature definition.""" + return self._featureDefinitions.get(featureCode) + + def getTemplateRoles(self, featureCode: str) -> List[Dict[str, Any]]: + """Get template roles for a feature.""" + return self._templateRoles.get(featureCode, []) + + def getAllTemplateRoles(self) -> Dict[str, List[Dict[str, Any]]]: + """Get all template roles grouped by feature.""" + return self._templateRoles.copy() + + def getRegisteredFeatures(self) -> List[str]: + """Get list of all registered feature codes.""" + return list(self._featureDefinitions.keys()) + + def unregisterFeature(self, featureCode: str) -> bool: + """Unregister all objects for a feature.""" + try: + for key in [k for k, v in self._uiObjects.items() if v["featureCode"] == featureCode]: + del self._uiObjects[key] + for key in [k for k, v in self._resourceObjects.items() if v["featureCode"] == featureCode]: + del self._resourceObjects[key] + self._featureDefinitions.pop(featureCode, None) + self._templateRoles.pop(featureCode, None) + logger.info(f"Unregistered feature: {featureCode}") + return True + except Exception as e: + logger.error(f"Failed to unregister feature {featureCode}: {e}") + return False + + def getCatalogStats(self) -> Dict[str, Any]: + """Get statistics about the catalog.""" + return { + "features": len(self._featureDefinitions), + "uiObjects": len(self._uiObjects), + "resourceObjects": len(self._resourceObjects), + "templateRoles": sum(len(roles) for roles in self._templateRoles.values()) + } + + +# Singleton accessor +_catalogService: Optional[RbacCatalogService] = None + + +def getCatalogService() -> RbacCatalogService: + """Get the singleton RBAC Catalog Service instance.""" + global _catalogService + if _catalogService is None: + _catalogService = RbacCatalogService() + return _catalogService diff --git a/modules/services/__init__.py b/modules/services/__init__.py index ff91dc6b..b5d124ff 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -1,17 +1,34 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -from typing import Any, Optional +""" +Services Module. +Central service registry that provides access to shared services. + +IMPORTANT: Import-Regelwerk +- Zentrale Module (wie dieses) dürfen KEINE Feature-Container importieren +- Feature-spezifische Services werden dynamisch geladen +- Nur Shared Services werden direkt geladen +""" + +import os +import importlib +import glob +from typing import Any, Optional, TYPE_CHECKING +import logging from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelChat import ChatWorkflow + +if TYPE_CHECKING: + from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow + +logger = logging.getLogger(__name__) + +# Path to feature containers +_FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features") + class PublicService: - """Lightweight proxy exposing only public callable attributes of a target. - - - Hides names starting with '_' - - Optionally restricts to callables only - - Optional name_filter predicate for allow-list patterns - """ + """Lightweight proxy exposing only public callable attributes of a target.""" def __init__(self, target: Any, functionsOnly: bool = True, nameFilter=None): self._target = target @@ -29,57 +46,44 @@ class PublicService: return attr def __dir__(self): - names = [ + return sorted([ n for n in dir(self._target) if not n.startswith('_') and (not self._functionsOnly or callable(getattr(self._target, n, None))) and (self._nameFilter(n) if self._nameFilter else True) - ] - return sorted(names) + ]) class Services: + """ + Central Services class providing access to all services. + + Import-Regelwerk: + - Shared Services are loaded directly (from modules/services/) + - Feature-specific Services are loaded dynamically via filename discovery + """ - def __init__(self, user: User, workflow: ChatWorkflow = None, mandateId: Optional[str] = None): + def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None): self.user: User = user - self.workflow: ChatWorkflow = workflow + self.workflow = workflow self.mandateId: Optional[str] = mandateId - self.currentUserPrompt: str = "" # Cleaned/normalized user intent for the current round - self.rawUserPrompt: str = "" # Original raw user message for the current round + self.currentUserPrompt: str = "" + self.rawUserPrompt: str = "" - # Initialize interfaces with explicit mandateId - - from modules.interfaces.interfaceDbChat import getInterface as getChatInterface - self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) - + # Initialize central interfaces from modules.interfaces.interfaceDbApp import getInterface as getAppInterface self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbTrustee import getInterface as getTrusteeInterface - self.interfaceDbTrustee = getTrusteeInterface(user, mandateId=mandateId) - - # Expose RBAC directly on services for convenience self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None - # Initialize service packages - - from .serviceExtraction.mainServiceExtraction import ExtractionService - self.extraction = PublicService(ExtractionService(self)) - - from .serviceGeneration.mainServiceGeneration import GenerationService - self.generation = PublicService(GenerationService(self)) - - from .serviceNeutralization.mainServiceNeutralization import NeutralizationService - self.neutralization = PublicService(NeutralizationService(self)) - + # ============================================================ + # SHARED SERVICES (from modules/services/) + # ============================================================ from .serviceSharepoint.mainServiceSharepoint import SharepointService self.sharepoint = PublicService(SharepointService(self)) - - from .serviceAi.mainServiceAi import AiService - self.ai = PublicService(AiService(self), functionsOnly=False) from .serviceTicket.mainServiceTicket import TicketService self.ticket = PublicService(TicketService(self)) @@ -90,25 +94,82 @@ class Services: from .serviceUtils.mainServiceUtils import UtilsService self.utils = PublicService(UtilsService(self)) - from .serviceWeb.mainServiceWeb import WebService - self.web = PublicService(WebService(self)) - from .serviceSecurity.mainServiceSecurity import SecurityService self.security = PublicService(SecurityService(self)) from .serviceMessaging.mainServiceMessaging import MessagingService self.messaging = PublicService(MessagingService(self)) - -def getInterface(user: User, workflow: ChatWorkflow = None, mandateId: Optional[str] = None) -> Services: - """ - Get Services instance for the given user and mandate context. + # ============================================================ + # FEATURE SERVICES (dynamically loaded by filename discovery) + # ============================================================ + self._loadFeatureInterfaces() + self._loadFeatureServices() - Args: - user: The authenticated user - workflow: Optional ChatWorkflow context - mandateId: Explicit mandate context (from RequestContext / X-Mandate-Id header). Required. - """ + def _loadFeatureInterfaces(self): + """Dynamically load interfaces from feature containers by filename pattern.""" + # Find all interfaceFeature*.py files + pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py") + for filepath in glob.glob(pattern): + try: + # Extract feature name and interface name + featureDir = os.path.basename(os.path.dirname(filepath)) + filename = os.path.basename(filepath)[:-3] # Remove .py + + # Build module path: modules.features.. + modulePath = f"modules.features.{featureDir}.{filename}" + module = importlib.import_module(modulePath) + + # Get interface via getInterface() + if hasattr(module, "getInterface"): + interface = module.getInterface(self.user, mandateId=self.mandateId) + # Derive attribute name: interfaceFeatureAiChat -> interfaceDbChat + attrName = filename.replace("interfaceFeature", "interfaceDb") + setattr(self, attrName, interface) + logger.debug(f"Loaded interface: {attrName} from {modulePath}") + except Exception as e: + logger.debug(f"Could not load interface from {filepath}: {e}") + + def _loadFeatureServices(self): + """Dynamically load services from feature containers by filename pattern.""" + # Find all service*/mainService*.py files in feature containers + pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py") + for filepath in glob.glob(pattern): + try: + # Extract paths + serviceDir = os.path.basename(os.path.dirname(filepath)) + featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath))) + filename = os.path.basename(filepath)[:-3] # Remove .py + + # Build module path: modules.features... + modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}" + module = importlib.import_module(modulePath) + + # Find service class (ends with "Service") + serviceClass = None + for name in dir(module): + if name.endswith("Service") and not name.startswith("_"): + cls = getattr(module, name) + if isinstance(cls, type): + serviceClass = cls + break + + if serviceClass: + # Derive attribute name: serviceAi -> ai, serviceExtraction -> extraction + attrName = serviceDir.replace("service", "").lower() + if not attrName: + attrName = serviceDir.lower() + + # Check if it needs functionsOnly=False (for AI service) + functionsOnly = attrName != "ai" + + serviceInstance = serviceClass(self) + setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly)) + logger.debug(f"Loaded service: {attrName} from {modulePath}") + except Exception as e: + logger.debug(f"Could not load service from {filepath}: {e}") + + +def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None) -> Services: + """Get Services instance for the given user and mandate context.""" return Services(user, workflow, mandateId=mandateId) - - diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 137dcd05..ec4cad6b 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.features.aichat.datamodelFeatureAiChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 2c975f1d..afdad31b 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -161,7 +161,7 @@ class UtilsService: Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChat. """ try: - from modules.interfaces.interfaceDbChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.features.aichat.interfaceFeatureAiChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/features/workflow/__init__.py b/modules/workflows/automation/__init__.py similarity index 100% rename from modules/features/workflow/__init__.py rename to modules/workflows/automation/__init__.py diff --git a/modules/features/workflow/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py similarity index 99% rename from modules/features/workflow/mainWorkflow.py rename to modules/workflows/automation/mainWorkflow.py index 70a2e9aa..43df551c 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/workflows/automation/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/workflows/automation/subAutomationSchedule.py b/modules/workflows/automation/subAutomationSchedule.py new file mode 100644 index 00000000..1061d65e --- /dev/null +++ b/modules/workflows/automation/subAutomationSchedule.py @@ -0,0 +1,64 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Automation Lifecycle Manager. +Handles startup and shutdown of scheduled automations. + +Note: This module is NOT for feature container lifecycle - it only manages +the automation scheduler (loading/syncing scheduled automation events). +""" + +import logging +from modules.services import getInterface as getServices + +logger = logging.getLogger(__name__) + + +async def start(eventUser) -> None: + """ + Start automation scheduler and sync scheduled events. + + Args: + eventUser: System-level event user for background operations (provided by app.py) + """ + if not eventUser: + logger.warning("Automation: No event user provided, skipping automation sync") + return True + + try: + from modules.workflows.automation import syncAutomationEvents + from modules.shared.callbackRegistry import callbackRegistry + + # Get services for event user (provides access to interfaces) + services = getServices(eventUser, None) + + # Register callback for automation changes + async def onAutomationChanged(chatInterface): + """Callback triggered when automations are created/updated/deleted.""" + eventServices = getServices(eventUser, None) + await syncAutomationEvents(eventServices, eventUser) + + callbackRegistry.register('automation.changed', onAutomationChanged) + logger.info("Automation: Registered change callback") + + # Initial sync on startup + await syncAutomationEvents(services, eventUser) + logger.info("Automation: Scheduled events synced on startup") + + except Exception as e: + logger.error(f"Automation: Error setting up events on startup: {str(e)}") + # Don't fail startup if automation sync fails + + return True + + +async def stop(eventUser) -> None: + """ + Stop automation scheduler. + + Args: + eventUser: System-level event user (provided by app.py) + """ + # Callbacks will remain registered (acceptable for shutdown) + logger.info("Automation: Scheduler stopped (callbacks cleaned up on shutdown)") + return True diff --git a/modules/workflows/automation/subAutomationTemplates.py b/modules/workflows/automation/subAutomationTemplates.py new file mode 100644 index 00000000..95c1eb77 --- /dev/null +++ b/modules/workflows/automation/subAutomationTemplates.py @@ -0,0 +1,385 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Automation templates for workflow definitions. + +Contains predefined workflow templates that can be used to create automation definitions. +""" + +from typing import Dict, Any, List + +# Automation templates structure +AUTOMATION_TEMPLATES: Dict[str, Any] = { + "sets": [ + { + "template": { + "overview": "SharePoint Themen Zusammenfassung", + "tasks": [ + { + "id": "Task01", + "title": "SharePoint Themen Zusammenfassung", + "description": "Erstellt eine Zusammenfassung aller SharePoint Sites und deren Inhalte", + "objective": "Erstelle eine Zusammenfassung aller SharePoint Themen (Sites) und deren Inhalte als Word-Dokument", + "actionList": [ + { + "execMethod": "sharepoint", + "execAction": "findDocumentPath", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "searchQuery": "*", + "maxResults": 100 + }, + "execResultLabel": "sharepoint_sites_found" + }, + { + "execMethod": "sharepoint", + "execAction": "listDocuments", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "pathQuery": "{{KEY:sharepointBasePath}}", + "includeSubfolders": True + }, + "execResultLabel": "sharepoint_structure" + }, + { + "execMethod": "ai", + "execAction": "process", + "execParameters": { + "aiPrompt": "{{KEY:summaryPrompt}}", + "documentList": ["sharepoint_sites_found", "sharepoint_structure"], + "resultType": "docx" + }, + "execResultLabel": "sharepoint_summary" + }, + { + "execMethod": "sharepoint", + "execAction": "uploadDocument", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "documentList": ["sharepoint_summary"], + "pathQuery": "{{KEY:sharepointFolderNameDestination}}" + }, + "execResultLabel": "sharepoint_upload_result" + } + ] + } + ] + }, + "parameters": { + "connectionName": "connection:msft:p.motsch@valueon.ch", + "sharepointBasePath": "/sites/company-share", + "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", + "summaryPrompt": "Erstelle eine umfassende Zusammenfassung aller SharePoint Sites und deren Inhalte. Strukturiere das Dokument nach Sites und fasse für jede Site die wichtigsten Themen, Ordnerstrukturen und Dokumente zusammen. Erstelle ein professionelles Word-Dokument mit Überschriften, Abschnitten und einer klaren Gliederung. Berücksichtige alle gefundenen Sites, deren Ordnerstrukturen und dokumentiere die wichtigsten Inhalte pro Site." + } + }, + { + "template": { + "overview": "Immobilienrecherche Zürich", + "tasks": [ + { + "id": "Task02", + "title": "Immobilienrecherche Zürich", + "description": "Webrecherche nach Immobilien im Kanton Zürich und Speicherung in Excel", + "objective": "Immobilienrecherche im Kanton Zürich zum Verkauf (5-20 Mio. CHF) und speichere Ergebnisse in Excel-Liste auf SharePoint", + "actionList": [ + { + "execMethod": "ai", + "execAction": "webResearch", + "execParameters": { + "prompt": "{{KEY:immobilienResearchPrompt}}", + "urlList": ["{{KEY:immobilienResearchUrl}}"] + }, + "execResultLabel": "immobilien_research_results" + }, + { + "execMethod": "ai", + "execAction": "process", + "execParameters": { + "aiPrompt": "{{KEY:excelFormatPrompt}}", + "documentList": ["immobilien_research_results"], + "resultType": "xlsx" + }, + "execResultLabel": "immobilien_excel_list" + }, + { + "execMethod": "sharepoint", + "execAction": "uploadDocument", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "documentList": ["immobilien_excel_list"], + "pathQuery": "{{KEY:sharepointFolderNameDestination}}" + }, + "execResultLabel": "immobilien_upload_result" + } + ] + } + ] + }, + "parameters": { + "connectionName": "connection:msft:p.motsch@valueon.ch", + "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", + "immobilienResearchUrl": ["https://www.homegate.ch", "https://www.immoscout24.ch", "https://www.immowelt.ch"], + "immobilienResearchPrompt": "Suche nach Immobilien zum Verkauf im Kanton Zürich, Schweiz, im Preisbereich von 5-20 Millionen CHF. Sammle Informationen zu: Ort, Preis, Beschreibung, URL zu Bildern, Verkäufer/Kontaktinformationen.", + "excelFormatPrompt": "Erstelle eine Excel-Datei mit den recherchierten Immobilien. Jede Immobilie soll eine Zeile sein mit den folgenden Spalten: Ort, Preis (in CHF), Beschreibung, URL zu Bild, Verkäufer. Verwende die Daten aus der Webrecherche." + } + }, + { + "template": { + "overview": "Spesenbelege Zusammenfassung", + "tasks": [ + { + "id": "Task03", + "title": "Spesenbelege CSV Zusammenfassung", + "description": "Liest PDF-Spesenbelege aus SharePoint-Ordner und erstellt CSV-Zusammenfassung", + "objective": "Extrahiere alle PDF-Spesenbelege aus einem SharePoint-Ordner und erstelle eine CSV-Datei mit allen Spesendaten im selben Ordner", + "actionList": [ + { + "execMethod": "sharepoint", + "execAction": "findDocumentPath", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "searchQuery": "{{KEY:sharepointFolderNameSource}}:files:.pdf", + "maxResults": 100 + }, + "execResultLabel": "sharepoint_pdf_files" + }, + { + "execMethod": "sharepoint", + "execAction": "readDocuments", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "pathObject": "sharepoint_pdf_files" + }, + "execResultLabel": "spesenbelege_documents" + }, + { + "execMethod": "ai", + "execAction": "process", + "execParameters": { + "aiPrompt": "{{KEY:expenseExtractionPrompt}}", + "documentList": ["spesenbelege_documents"], + "resultType": "csv" + }, + "execResultLabel": "spesenbelege_csv" + }, + { + "execMethod": "sharepoint", + "execAction": "uploadDocument", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "documentList": ["spesenbelege_csv"], + "pathQuery": "{{KEY:sharepointFolderNameDestination}}" + }, + "execResultLabel": "spesenbelege_upload_result" + } + ] + } + ] + }, + "parameters": { + "connectionName": "connection:msft:p.motsch@valueon.ch", + "sharepointFolderNameSource": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/expenses", + "sharepointFolderNameDestination": "/sites/company-share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output", + "expenseExtractionPrompt": "Verarbeite alle bereitgestellten Dokumente, aber extrahiere nur Daten aus PDF-Spesenbelegen (ignoriere andere Dateitypen). Für jeden gefundenen PDF-Spesenbeleg extrahiere als separaten Datensatz: Datum, Betrag, MWST %, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler/Verkäufer, Steuerbetrag. Erstelle eine CSV-Datei mit einer Zeile pro Spesenbeleg. Verwende die folgenden Spaltenüberschriften: Datum, Betrag, Währung, Kategorie, Beschreibung, Rechnungsnummer, Händler, Steuerbetrag. Stelle sicher, dass alle Beträge numerisch sind und Datumswerte im Format YYYY-MM-DD vorliegen. Wenn ein Dokument kein Spesenbeleg ist, ignoriere es." + } + }, + { + "template": { + "overview": "Preprocessing Server Data Update", + "tasks": [ + { + "id": "Task04", + "title": "Trigger Preprocessing Server", + "description": "Triggers the preprocessing server at customer tenant to update database with configuration", + "objective": "Call preprocessing server endpoint to update database with provided configuration JSON", + "actionList": [ + { + "execMethod": "context", + "execAction": "triggerPreprocessingServer", + "execParameters": { + "endpoint": "{{KEY:endpoint}}", + "configJson": "{{KEY:configJson}}", + "authSecretConfigKey": "{{KEY:authSecretConfigKey}}" + }, + "execResultLabel": "preprocessing_server_result" + } + ] + } + ] + }, + "parameters": { + "endpoint": "https://poweron-althaus-preprocess-prod-e3fegaatc7faency.switzerlandnorth-01.azurewebsites.net/api/v1/dataprocessor/update-db-with-config", + "authSecretConfigKey": "PREPROCESS_ALTHAUS_CHAT_SECRET", + "configJson": "{\"tables\":[{\"name\":\"Artikel\",\"powerbi_table_name\":\"Artikel\",\"steps\":[{\"keep\":{\"columns\":[\"I_ID\",\"Artikelbeschrieb\",\"Artikelbezeichnung\",\"Artikelgruppe\",\"Artikelkategorie\",\"Artikelkürzel\",\"Artikelnummer\",\"Einheit\",\"Gesperrt\",\"Keywords\",\"Lieferant\",\"Warengruppe\"]}},{\"fillna\":{\"column\":\"Lieferant\",\"value\":\"Unbekannt\"}}]},{\"name\":\"Einkaufspreis\",\"powerbi_table_name\":\"Einkaufspreis\",\"steps\":[{\"to_numeric\":{\"column\":\"EP_CHF\",\"errors\":\"coerce\"}},{\"dropna\":{\"subset\":[\"EP_CHF\"]}}]}]}" + } + }, + { + "template": { + "overview": "JIRA to SharePoint Ticket Synchronization", + "tasks": [ + { + "id": "Task01", + "title": "Sync JIRA Tickets to SharePoint", + "description": "Export JIRA tickets, merge with SharePoint file, upload back, and import changes to JIRA", + "objective": "Synchronize JIRA tickets with SharePoint file (bidirectional sync)", + "actionList": [ + { + "execMethod": "sharepoint", + "execAction": "findSiteByUrl", + "execParameters": { + "connectionReference": "{{KEY:sharepointConnection}}", + "hostname": "{{KEY:sharepointHostname}}", + "sitePath": "{{KEY:sharepointSitePath}}" + }, + "execResultLabel": "sharepoint_site" + }, + { + "execMethod": "jira", + "execAction": "connectJira", + "execParameters": { + "apiUsername": "{{KEY:jiraUsername}}", + "apiTokenConfigKey": "{{KEY:jiraTokenConfigKey}}", + "apiUrl": "{{KEY:jiraUrl}}", + "projectCode": "{{KEY:jiraProjectCode}}", + "issueType": "{{KEY:jiraIssueType}}", + "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" + }, + "execResultLabel": "jira_connection" + }, + { + "execMethod": "jira", + "execAction": "exportTicketsAsJson", + "execParameters": { + "connectionId": "jira_connection", + "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" + }, + "execResultLabel": "jira_exported_tickets" + }, + { + "execMethod": "sharepoint", + "execAction": "downloadFileByPath", + "execParameters": { + "connectionReference": "{{KEY:sharepointConnection}}", + "siteId": "sharepoint_site", + "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" + }, + "execResultLabel": "existing_file_content" + }, + { + "execMethod": "jira", + "execAction": "parseExcelContent", + "execParameters": { + "excelContent": "existing_file_content", + "skipRows": 3, + "hasCustomHeaders": True + }, + "execResultLabel": "existing_parsed_data" + }, + { + "execMethod": "jira", + "execAction": "mergeTicketData", + "execParameters": { + "jiraData": "jira_exported_tickets", + "existingData": "existing_parsed_data", + "taskSyncDefinition": "{{KEY:taskSyncDefinition}}", + "idField": "ID" + }, + "execResultLabel": "merged_ticket_data" + }, + { + "execMethod": "sharepoint", + "execAction": "copyFile", + "execParameters": { + "connectionReference": "{{KEY:sharepointConnection}}", + "siteId": "sharepoint_site", + "sourceFolder": "{{KEY:sharepointMainFolder}}", + "sourceFile": "{{KEY:syncFileName}}", + "destFolder": "{{KEY:sharepointBackupFolder}}", + "destFile": "backup_{{TIMESTAMP}}_{{KEY:syncFileName}}" + }, + "execResultLabel": "file_backup" + }, + { + "execMethod": "jira", + "execAction": "createExcelContent", + "execParameters": { + "data": "merged_ticket_data", + "headers": "existing_parsed_data", + "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" + }, + "execResultLabel": "new_file_content" + }, + { + "execMethod": "sharepoint", + "execAction": "uploadFile", + "execParameters": { + "connectionReference": "{{KEY:sharepointConnection}}", + "siteId": "sharepoint_site", + "folderPath": "{{KEY:sharepointMainFolder}}", + "fileName": "{{KEY:syncFileName}}", + "content": "new_file_content" + }, + "execResultLabel": "uploaded_file" + }, + { + "execMethod": "sharepoint", + "execAction": "downloadFileByPath", + "execParameters": { + "connectionReference": "{{KEY:sharepointConnection}}", + "siteId": "sharepoint_site", + "filePath": "{{KEY:sharepointMainFolder}}/{{KEY:syncFileName}}" + }, + "execResultLabel": "uploaded_file_content" + }, + { + "execMethod": "jira", + "execAction": "parseExcelContent", + "execParameters": { + "excelContent": "uploaded_file_content", + "skipRows": 3, + "hasCustomHeaders": True + }, + "execResultLabel": "import_data" + }, + { + "execMethod": "jira", + "execAction": "importTicketsFromJson", + "execParameters": { + "connectionId": "jira_connection", + "ticketData": "import_data", + "taskSyncDefinition": "{{KEY:taskSyncDefinition}}" + }, + "execResultLabel": "import_result" + } + ] + } + ] + }, + "parameters": { + "sharepointConnection": "connection:msft:patrick.motsch@delta.ch", + "sharepointHostname": "deltasecurityag.sharepoint.com", + "sharepointSitePath": "SteeringBPM", + "sharepointMainFolder": "/General/50 Docs hosted by SELISE", + "sharepointBackupFolder": "/General/50 Docs hosted by SELISE/SyncHistory", + "syncFileName": "DELTAgroup x SELISE Ticket Exchange List.xlsx", + "jiraUsername": "p.motsch@valueon.ch", + "jiraTokenConfigKey": "Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET", + "jiraUrl": "https://deltasecurity.atlassian.net", + "jiraProjectCode": "DCS", + "jiraIssueType": "Task", + "taskSyncDefinition": "{\"ID\":[\"get\",[\"key\"]],\"Module Category\":[\"get\",[\"fields\",\"customfield_10058\",\"value\"]],\"Summary\":[\"get\",[\"fields\",\"summary\"]],\"Description\":[\"get\",[\"fields\",\"description\"]],\"References\":[\"get\",[\"fields\",\"customfield_10066\"]],\"Priority\":[\"get\",[\"fields\",\"priority\",\"name\"]],\"Issue Status\":[\"get\",[\"fields\",\"status\",\"name\"]],\"Assignee\":[\"get\",[\"fields\",\"assignee\",\"displayName\"]],\"Issue Created\":[\"get\",[\"fields\",\"created\"]],\"Due Date\":[\"get\",[\"fields\",\"duedate\"]],\"DELTA Comments\":[\"get\",[\"fields\",\"customfield_10167\"]],\"SELISE Ticket References\":[\"put\",[\"fields\",\"customfield_10067\"]],\"SELISE Status Values\":[\"put\",[\"fields\",\"customfield_10065\"]],\"SELISE Comments\":[\"put\",[\"fields\",\"customfield_10168\"]]}" + } + } + ] +} + + +def getAutomationTemplates() -> Dict[str, Any]: + """ + Get automation templates. + + Returns: + Dict containing the automation templates structure with 'sets' key. + """ + return AUTOMATION_TEMPLATES + diff --git a/modules/workflows/automation/subAutomationUtils.py b/modules/workflows/automation/subAutomationUtils.py new file mode 100644 index 00000000..97d28719 --- /dev/null +++ b/modules/workflows/automation/subAutomationUtils.py @@ -0,0 +1,118 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Utility functions for automation feature. + +Moved from interfaces/interfaceDbChat.py. +""" + +import json +from typing import Dict, Any +from datetime import datetime, UTC + + +def parseScheduleToCron(schedule: str) -> Dict[str, Any]: + """Parse schedule string to cron kwargs for APScheduler""" + parts = schedule.split() + if len(parts) != 5: + raise ValueError(f"Invalid schedule format: {schedule}") + + return { + "minute": parts[0], + "hour": parts[1], + "day": parts[2], + "month": parts[3], + "day_of_week": parts[4] + } + + +def planToPrompt(plan: Dict) -> str: + """Convert plan structure to prompt string for workflow execution""" + return plan.get("userMessage", plan.get("overview", "Execute automation workflow")) + + +def replacePlaceholders(template: str, placeholders: Dict[str, str]) -> str: + """Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}} or {{TIMESTAMP}}""" + result = template + + # Replace TIMESTAMP placeholder first (calculated placeholder, not from parameters) + timestampPattern = "{{TIMESTAMP}}" + if timestampPattern in result: + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + result = result.replace(timestampPattern, timestamp) + + for placeholderName, value in placeholders.items(): + pattern = f"{{{{KEY:{placeholderName}}}}}" + + # Check if placeholder is in an array context like ["{{KEY:...}}"] + # If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array + arrayPattern = f'["{pattern}"]' + if arrayPattern in result: + # Check if value is a JSON array/dict + isArrayValue = False + arrayValue = None + + if isinstance(value, (list, dict)): + isArrayValue = True + arrayValue = json.dumps(value) + elif isinstance(value, str): + try: + parsed = json.loads(value) + if isinstance(parsed, (list, dict)): + isArrayValue = True + arrayValue = value # Already valid JSON string + except (json.JSONDecodeError, ValueError): + pass + + if isArrayValue: + # Replace ["{{KEY:...}}"] with the array value + result = result.replace(arrayPattern, arrayValue) + continue # Skip the regular replacement below + + # Regular replacement - check if in quoted context + patternStart = result.find(pattern) + isQuoted = False + if patternStart > 0: + charBefore = result[patternStart - 1] if patternStart > 0 else None + patternEnd = patternStart + len(pattern) + charAfter = result[patternEnd] if patternEnd < len(result) else None + if charBefore == '"' and charAfter == '"': + isQuoted = True + + # Handle different value types + if isinstance(value, (list, dict)): + # Python list/dict - convert to JSON + replacement = json.dumps(value) + elif isinstance(value, str): + # String value - check if it's a JSON string representing list/dict + try: + parsed = json.loads(value) + if isinstance(parsed, (list, dict)): + # It's a JSON string of a list/dict + if isQuoted: + # In quoted context, escape the JSON string + escaped = json.dumps(value) + replacement = escaped[1:-1] # Remove outer quotes + else: + # In unquoted context, use JSON directly + replacement = value + else: + # It's a JSON string of a primitive + if isQuoted: + escaped = json.dumps(value) + replacement = escaped[1:-1] + else: + replacement = value + except (json.JSONDecodeError, ValueError): + # Not valid JSON - treat as plain string + if isQuoted: + escaped = json.dumps(value) + replacement = escaped[1:-1] + else: + replacement = value + else: + # Numbers, booleans, None - convert to string + replacement = str(value) + result = result.replace(pattern, replacement) + return result + diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index 39d6e16f..03f014d5 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.features.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 4f9bbd21..fecba4e9 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 65e95a32..dc3231df 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index f804c0b9..06a4817f 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index e32c1965..8dd37872 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.features.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index bb6f8437..0e04dde8 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.features.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 62b43bce..8b462e62 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index ff7e896f..b0cc5a0e 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index 5b90ce13..dabd2f06 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index 9991285b..c46c9924 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 8e3b7185..082497b9 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index 2f011a25..c7c94e6c 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index 45b60cad..c6e2b69a 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index cbec7960..a33278f3 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index 631795b3..a4491efb 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 55d99654..57179415 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index b997889e..60645a06 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 2bd7ab74..899c49d3 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index bbdc2cc7..455592eb 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index 5ac4e548..b7cf9214 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 59604896..3d8ec9d6 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 2d325d9f..1fe4992a 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index f8831d59..0d7c1b6b 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index 9b7fb011..9f08ddae 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index a4bf18b6..97d42867 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index f149e482..7855627d 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index c64a6637..34aa41e7 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 722dbc99..88c07269 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index 62b6dd94..58d0a2e0 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index 318271c3..2c6bde6b 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index 73cdb730..7059f1d0 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index e9361853..bdabdcbb 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 1f469b80..362c7ba7 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 0e4d6ee4..b621dbff 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionItem, TaskStep +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index a4ae05e9..84e0c24f 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 0fac427c..af2726b7 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan +from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.datamodels.datamodelChat import WorkflowModeEnum + from modules.features.aichat.datamodelFeatureAiChat import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index e3131939..085ff694 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelChat import ( +from modules.features.aichat.datamodelFeatureAiChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index 770c868a..de6016ec 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskResult, ActionItem +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index f7754eab..606a6ce3 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.datamodels.datamodelChat import ( +from modules.features.aichat.datamodelFeatureAiChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.datamodels.datamodelChat import ReviewContext + from modules.features.aichat.datamodelFeatureAiChat import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.datamodels.datamodelChat import ReviewResult + from modules.features.aichat.datamodelFeatureAiChat import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 1cdf0d53..2094d07a 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation +from modules.features.aichat.datamodelFeatureAiChat import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 136dd2cb..260ef8b8 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.features.aichat.datamodelFeatureAiChat import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.features.aichat.datamodelFeatureAiChat import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict @@ -452,7 +452,7 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.interfaces.interfaceDbChat as interfaceDbChat + import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() interfaceDbChat = interfaceDbChat.getInterface(rootInterface.currentUser) diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 31878033..92432038 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.features.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index 11a54ca1..d840cc1e 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.features.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 9c9d6c84..b613d5fd 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -7,8 +7,8 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING from modules.datamodels import datamodelChat -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.datamodels.datamodelChat import ActionDocument + from modules.features.aichat.datamodelFeatureAiChat import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.datamodels.datamodelChat import ChatMessage, ChatDocument, ActionDocument + from modules.features.aichat.datamodelFeatureAiChat import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index a9b656eb..a7d28d39 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.datamodels.datamodelChat import ( +from modules.features.aichat.datamodelFeatureAiChat import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.datamodels.datamodelChat import TaskContext +from modules.features.aichat.datamodelFeatureAiChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.features.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.features.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.datamodels.datamodelChat import ActionResult + from modules.features.aichat.datamodelFeatureAiChat import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/scripts/import_analysis.csv b/scripts/import_analysis.csv new file mode 100644 index 00000000..a81312fe --- /dev/null +++ b/scripts/import_analysis.csv @@ -0,0 +1,2716 @@ +module_name,imported_module_name,position,import_valid +gateway.app,contextlib,header,Yes +gateway.app,datetime,header,Yes +gateway.app,fastapi,header,Yes +gateway.app,fastapi.middleware.cors,header,Yes +gateway.app,fastapi.openapi.utils,function customOpenapi,Yes +gateway.app,fastapi.security,header,Yes +gateway.app,logging,header,Yes +gateway.app,logging.handlers,header,Yes +gateway.app,modules.auth,header,Yes +gateway.app,modules.auth,header,Yes +gateway.app,modules.features.featureRegistry,header,Yes +gateway.app,modules.features.featureRegistry,header,Yes +gateway.app,modules.interfaces.interfaceDbApp,header,Yes +gateway.app,modules.routes.routeAdmin,header,Yes +gateway.app,modules.routes.routeAdminAutomationEvents,header,Yes +gateway.app,modules.routes.routeAdminFeatures,header,Yes +gateway.app,modules.routes.routeAdminRbacExport,header,Yes +gateway.app,modules.routes.routeAdminRbacRules,header,Yes +gateway.app,modules.routes.routeAttributes,header,Yes +gateway.app,modules.routes.routeDataConnections,header,Yes +gateway.app,modules.routes.routeDataFiles,header,Yes +gateway.app,modules.routes.routeDataMandates,header,Yes +gateway.app,modules.routes.routeDataPrompts,header,Yes +gateway.app,modules.routes.routeDataUsers,header,Yes +gateway.app,modules.routes.routeDataWorkflows,header,Yes +gateway.app,modules.routes.routeGdpr,header,Yes +gateway.app,modules.routes.routeInvitations,header,Yes +gateway.app,modules.routes.routeMessaging,header,Yes +gateway.app,modules.routes.routeOptions,header,Yes +gateway.app,modules.routes.routeSecurityAdmin,header,Yes +gateway.app,modules.routes.routeSecurityGoogle,header,Yes +gateway.app,modules.routes.routeSecurityLocal,header,Yes +gateway.app,modules.routes.routeSecurityMsft,header,Yes +gateway.app,modules.routes.routeSharepoint,header,Yes +gateway.app,modules.routes.routeVoiceGoogle,header,Yes +gateway.app,modules.shared.auditLogger,function lifespan,Yes +gateway.app,modules.shared.configuration,header,Yes +gateway.app,modules.shared.eventManagement,header,Yes +gateway.app,modules.workflows.automation,header,Yes +gateway.app,os,header,Yes +gateway.app,sys,header,Yes +gateway.app,unicodedata,header,Yes +gateway.app,urllib.parse,header,Yes +gateway.modules.auth.__init__,(relative) .authentication,header,Yes +gateway.modules.auth.__init__,(relative) .csrf,header,Yes +gateway.modules.auth.__init__,(relative) .jwtService,header,Yes +gateway.modules.auth.__init__,(relative) .tokenManager,header,Yes +gateway.modules.auth.__init__,(relative) .tokenRefreshMiddleware,header,Yes +gateway.modules.auth.__init__,(relative) .tokenRefreshService,header,Yes +gateway.modules.auth.authentication,fastapi,header,Yes +gateway.modules.auth.authentication,fastapi.security,header,Yes +gateway.modules.auth.authentication,jose,header,Yes +gateway.modules.auth.authentication,logging,header,Yes +gateway.modules.auth.authentication,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.auth.authentication,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.auth.authentication,modules.datamodels.datamodelUam,header,Yes +gateway.modules.auth.authentication,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.auth.authentication,modules.security.rootAccess,header,Yes +gateway.modules.auth.authentication,modules.shared.auditLogger,function requireSysAdmin,Yes +gateway.modules.auth.authentication,modules.shared.configuration,header,Yes +gateway.modules.auth.authentication,slowapi,header,Yes +gateway.modules.auth.authentication,slowapi.util,header,Yes +gateway.modules.auth.authentication,typing,header,Yes +gateway.modules.auth.csrf,fastapi,header,Yes +gateway.modules.auth.csrf,fastapi.responses,function dispatch,Yes +gateway.modules.auth.csrf,fastapi.responses,function dispatch,Yes +gateway.modules.auth.csrf,logging,header,Yes +gateway.modules.auth.csrf,starlette.middleware.base,header,Yes +gateway.modules.auth.csrf,typing,header,Yes +gateway.modules.auth.jwtService,datetime,header,Yes +gateway.modules.auth.jwtService,fastapi,header,Yes +gateway.modules.auth.jwtService,jose,header,Yes +gateway.modules.auth.jwtService,modules.shared.configuration,header,Yes +gateway.modules.auth.jwtService,modules.shared.timeUtils,header,Yes +gateway.modules.auth.jwtService,typing,header,Yes +gateway.modules.auth.jwtService,uuid,function createAccessToken,Yes +gateway.modules.auth.jwtService,uuid,function createRefreshToken,Yes +gateway.modules.auth.tokenManager,httpx,header,Yes +gateway.modules.auth.tokenManager,logging,header,Yes +gateway.modules.auth.tokenManager,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.auth.tokenManager,modules.datamodels.datamodelUam,header,Yes +gateway.modules.auth.tokenManager,modules.interfaces.interfaceDbApp,function getFreshToken,Yes +gateway.modules.auth.tokenManager,modules.security.rootAccess,function getFreshToken,Yes +gateway.modules.auth.tokenManager,modules.shared.configuration,header,Yes +gateway.modules.auth.tokenManager,modules.shared.timeUtils,header,Yes +gateway.modules.auth.tokenManager,typing,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,asyncio,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,fastapi,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,logging,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,modules.auth.tokenRefreshService,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,modules.shared.timeUtils,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,starlette.middleware.base,header,Yes +gateway.modules.auth.tokenRefreshMiddleware,typing,header,Yes +gateway.modules.auth.tokenRefreshService,logging,header,Yes +gateway.modules.auth.tokenRefreshService,modules.auth.tokenManager,function _refresh_google_token,Yes +gateway.modules.auth.tokenRefreshService,modules.auth.tokenManager,function _refresh_microsoft_token,Yes +gateway.modules.auth.tokenRefreshService,modules.datamodels.datamodelUam,header,Yes +gateway.modules.auth.tokenRefreshService,modules.interfaces.interfaceDbApp,function refresh_expired_tokens,Yes +gateway.modules.auth.tokenRefreshService,modules.interfaces.interfaceDbApp,function proactive_refresh,Yes +gateway.modules.auth.tokenRefreshService,modules.security.rootAccess,function refresh_expired_tokens,Yes +gateway.modules.auth.tokenRefreshService,modules.security.rootAccess,function proactive_refresh,Yes +gateway.modules.auth.tokenRefreshService,modules.shared.auditLogger,header,Yes +gateway.modules.auth.tokenRefreshService,modules.shared.timeUtils,header,Yes +gateway.modules.auth.tokenRefreshService,typing,header,Yes +gateway.modules.connectors.connectorDbPostgre,json,function _save_record,Yes +gateway.modules.connectors.connectorDbPostgre,json,function _loadRecord,Yes +gateway.modules.connectors.connectorDbPostgre,json,function _loadTable,Yes +gateway.modules.connectors.connectorDbPostgre,json,function getRecordset,Yes +gateway.modules.connectors.connectorDbPostgre,logging,header,Yes +gateway.modules.connectors.connectorDbPostgre,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.connectors.connectorDbPostgre,modules.datamodels.datamodelUam,header,Yes +gateway.modules.connectors.connectorDbPostgre,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorDbPostgre,modules.shared.timeUtils,header,Yes +gateway.modules.connectors.connectorDbPostgre,psycopg2,header,Yes +gateway.modules.connectors.connectorDbPostgre,psycopg2.extras,header,Yes +gateway.modules.connectors.connectorDbPostgre,pydantic,header,Yes +gateway.modules.connectors.connectorDbPostgre,threading,header,Yes +gateway.modules.connectors.connectorDbPostgre,typing,header,Yes +gateway.modules.connectors.connectorDbPostgre,uuid,header,Yes +gateway.modules.connectors.connectorMessagingEmail,azure.communication.email,header,Yes +gateway.modules.connectors.connectorMessagingEmail,logging,header,Yes +gateway.modules.connectors.connectorMessagingEmail,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorMessagingEmail,typing,header,Yes +gateway.modules.connectors.connectorMessagingSms,logging,header,Yes +gateway.modules.connectors.connectorMessagingSms,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorMessagingSms,twilio.rest,function __init__,Yes +gateway.modules.connectors.connectorMessagingSms,typing,header,Yes +gateway.modules.connectors.connectorPreprocessor,httpx,header,Yes +gateway.modules.connectors.connectorPreprocessor,logging,header,Yes +gateway.modules.connectors.connectorPreprocessor,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorPreprocessor,typing,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,aiohttp,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,asyncio,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,logging,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,re,header,Yes +gateway.modules.connectors.connectorSwissTopoMapServer,typing,header,Yes +gateway.modules.connectors.connectorTicketsClickup,aiohttp,header,Yes +gateway.modules.connectors.connectorTicketsClickup,logging,header,Yes +gateway.modules.connectors.connectorTicketsClickup,modules.datamodels.datamodelTickets,header,Yes +gateway.modules.connectors.connectorTicketsClickup,typing,header,Yes +gateway.modules.connectors.connectorTicketsJira,aiohttp,header,Yes +gateway.modules.connectors.connectorTicketsJira,asyncio,header,Yes +gateway.modules.connectors.connectorTicketsJira,json,header,Yes +gateway.modules.connectors.connectorTicketsJira,logging,header,Yes +gateway.modules.connectors.connectorTicketsJira,modules.datamodels.datamodelTickets,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,google.cloud,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,google.cloud,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,google.cloud,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,google.oauth2,function __init__,Yes +gateway.modules.connectors.connectorVoiceGoogle,html,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,json,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,logging,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,modules.shared.configuration,header,Yes +gateway.modules.connectors.connectorVoiceGoogle,typing,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.__init__,(relative) .,header,Yes +gateway.modules.datamodels.datamodelAi,enum,header,Yes +gateway.modules.datamodels.datamodelAi,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.datamodels.datamodelAi,pydantic,header,Yes +gateway.modules.datamodels.datamodelAi,typing,header,Yes +gateway.modules.datamodels.datamodelAudit,enum,header,Yes +gateway.modules.datamodels.datamodelAudit,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelAudit,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelAudit,pydantic,header,Yes +gateway.modules.datamodels.datamodelAudit,typing,header,Yes +gateway.modules.datamodels.datamodelAudit,uuid,header,Yes +gateway.modules.datamodels.datamodelDocref,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelDocref,pydantic,header,Yes +gateway.modules.datamodels.datamodelDocref,typing,header,Yes +gateway.modules.datamodels.datamodelDocument,datetime,header,Yes +gateway.modules.datamodels.datamodelDocument,pydantic,header,Yes +gateway.modules.datamodels.datamodelDocument,typing,header,Yes +gateway.modules.datamodels.datamodelExtraction,pydantic,header,Yes +gateway.modules.datamodels.datamodelExtraction,typing,header,Yes +gateway.modules.datamodels.datamodelFeatures,modules.datamodels.datamodelUtils,header,Yes +gateway.modules.datamodels.datamodelFeatures,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelFeatures,pydantic,header,Yes +gateway.modules.datamodels.datamodelFeatures,typing,header,Yes +gateway.modules.datamodels.datamodelFeatures,uuid,header,Yes +gateway.modules.datamodels.datamodelFiles,base64,header,Yes +gateway.modules.datamodels.datamodelFiles,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelFiles,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelFiles,pydantic,header,Yes +gateway.modules.datamodels.datamodelFiles,typing,header,Yes +gateway.modules.datamodels.datamodelFiles,uuid,header,Yes +gateway.modules.datamodels.datamodelInvitation,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelInvitation,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelInvitation,pydantic,header,Yes +gateway.modules.datamodels.datamodelInvitation,secrets,header,Yes +gateway.modules.datamodels.datamodelInvitation,typing,header,Yes +gateway.modules.datamodels.datamodelInvitation,uuid,header,Yes +gateway.modules.datamodels.datamodelJson,typing,header,Yes +gateway.modules.datamodels.datamodelMembership,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelMembership,pydantic,header,Yes +gateway.modules.datamodels.datamodelMembership,uuid,header,Yes +gateway.modules.datamodels.datamodelMessaging,enum,header,Yes +gateway.modules.datamodels.datamodelMessaging,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelMessaging,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelMessaging,pydantic,header,Yes +gateway.modules.datamodels.datamodelMessaging,typing,header,Yes +gateway.modules.datamodels.datamodelMessaging,uuid,header,Yes +gateway.modules.datamodels.datamodelPagination,math,header,Yes +gateway.modules.datamodels.datamodelPagination,pydantic,header,Yes +gateway.modules.datamodels.datamodelPagination,typing,header,Yes +gateway.modules.datamodels.datamodelRbac,enum,header,Yes +gateway.modules.datamodels.datamodelRbac,modules.datamodels.datamodelUam,header,Yes +gateway.modules.datamodels.datamodelRbac,modules.datamodels.datamodelUtils,header,Yes +gateway.modules.datamodels.datamodelRbac,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelRbac,pydantic,header,Yes +gateway.modules.datamodels.datamodelRbac,typing,header,Yes +gateway.modules.datamodels.datamodelRbac,uuid,header,Yes +gateway.modules.datamodels.datamodelSecurity,(relative) .datamodelUam,header,Yes +gateway.modules.datamodels.datamodelSecurity,enum,header,Yes +gateway.modules.datamodels.datamodelSecurity,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelSecurity,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelSecurity,pydantic,header,Yes +gateway.modules.datamodels.datamodelSecurity,typing,header,Yes +gateway.modules.datamodels.datamodelSecurity,uuid,header,Yes +gateway.modules.datamodels.datamodelTickets,abc,header,Yes +gateway.modules.datamodels.datamodelTickets,pydantic,header,Yes +gateway.modules.datamodels.datamodelTickets,typing,header,Yes +gateway.modules.datamodels.datamodelUam,enum,header,Yes +gateway.modules.datamodels.datamodelUam,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelUam,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelUam,pydantic,header,Yes +gateway.modules.datamodels.datamodelUam,typing,header,Yes +gateway.modules.datamodels.datamodelUam,uuid,header,Yes +gateway.modules.datamodels.datamodelUtils,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelUtils,pydantic,header,Yes +gateway.modules.datamodels.datamodelUtils,typing,header,Yes +gateway.modules.datamodels.datamodelUtils,uuid,header,Yes +gateway.modules.datamodels.datamodelVoice,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelVoice,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelVoice,pydantic,header,Yes +gateway.modules.datamodels.datamodelVoice,uuid,header,Yes +gateway.modules.datamodels.datamodelWorkflow,modules.datamodels.datamodelDocref,header,Yes +gateway.modules.datamodels.datamodelWorkflow,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelWorkflow,modules.shared.jsonUtils,header,Yes +gateway.modules.datamodels.datamodelWorkflow,pydantic,header,Yes +gateway.modules.datamodels.datamodelWorkflow,typing,header,Yes +gateway.modules.datamodels.datamodelWorkflowActions,modules.datamodels.datamodelChat,header,No +gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.frontendTypes,header,Yes +gateway.modules.datamodels.datamodelWorkflowActions,pydantic,header,Yes +gateway.modules.datamodels.datamodelWorkflowActions,typing,header,Yes +gateway.modules.features.aichat.aicore.aicoreBase,abc,header,Yes +gateway.modules.features.aichat.aicore.aicoreBase,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicoreBase,time,function getCachedModels,Yes +gateway.modules.features.aichat.aicore.aicoreBase,typing,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,importlib,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,logging,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.security.rbac,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.security.rbacHelpers,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,os,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,time,function refreshModels,Yes +gateway.modules.features.aichat.aicore.aicoreModelRegistry,typing,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelSelector,logging,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelSelector,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicoreModelSelector,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,base64,function callAiImage,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,fastapi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,httpx,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,logging,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,modules.shared.configuration,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,os,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,time,function callAiImage,Yes +gateway.modules.features.aichat.aicore.aicorePluginAnthropic,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginInternal,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginInternal,logging,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginInternal,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginInternal,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,fastapi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,httpx,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,json,function generateImage,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,logging,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,function generateImage,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.shared.configuration,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginOpenai,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,fastapi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,httpx,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webSearch,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,logging,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,function _testConnection,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelTools,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.shared.configuration,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginPerplexity,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,(relative) .aicoreBase,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,asyncio,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,dataclasses,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,logging,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelTools,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.shared.configuration,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,re,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,re,function _cleanUrl,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,tavily,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,typing,header,Yes +gateway.modules.features.aichat.aicore.aicorePluginTavily,urllib.parse,function _normalizeUrl,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,enum,header,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,modules.shared.attributeUtils,header,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,modules.shared.timeUtils,header,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,pydantic,header,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,typing,header,Yes +gateway.modules.features.aichat.datamodelFeatureAiChat,uuid,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,asyncio,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,datetime,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,json,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,logging,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,math,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.security.rbac,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.configuration,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.timeUtils,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,os,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,typing,header,Yes +gateway.modules.features.aichat.interfaceFeatureAiChat,uuid,header,Yes +gateway.modules.features.aichat.mainAiChat,(relative) .aicore.aicoreModelRegistry,function onStart,Yes +gateway.modules.features.aichat.mainAiChat,logging,header,Yes +gateway.modules.features.aichat.mainAiChat,typing,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,(relative) .,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,fastapi,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,logging,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,modules.auth,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,modules.workflows.automation,header,Yes +gateway.modules.features.aichat.routeFeatureAiChat,typing,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subAiCallLooping,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subContentExtraction,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subDocumentIntents,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subResponseParsing,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subStructureFilling,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subStructureGeneration,function _initializeSubmodules,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,base64,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,json,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,logging,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceExtraction.mainServiceExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.mainServiceGeneration,function renderResult,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.codePath,function _handleCodeGeneration,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.documentPath,function _handleDocumentGeneration,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.imagePath,function _handleImageGeneration,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.interfaces.interfaceAiObjects,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,re,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,time,header,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,time,function _handleDataExtraction,Yes +gateway.modules.features.aichat.serviceAi.mainServiceAi,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,(relative) .subLoopingUseCases,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,json,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonContinuation,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceAi.subAiCallLooping,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,base64,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,json,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function extractTextFromImage,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function processTextContentWithAi,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,function extractAndPrepareContent,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,traceback,function extractTextFromImage,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,traceback,function processTextContentWithAi,Yes +gateway.modules.features.aichat.serviceAi.subContentExtraction,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,json,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,function resolvePreExtractedDocument,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,traceback,function resolvePreExtractedDocument,Yes +gateway.modules.features.aichat.serviceAi.subDocumentIntents,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,datetime,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,json,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,os,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,re,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonMerger,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,(relative) .subJsonMerger,function mergeJsonStringsWithOverlap,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,json,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.debugLogger,function mergeFragmentIntoSection,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function mergeJsonStringsWithOverlap,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _normalizeToElementsStructure,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractRowsFromFragment,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractOverlapAndContinuation,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeWithExplicitOverlap,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _smartConcatenate,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStringsWithOverlapFallback,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStructuresGeneric,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function extractKpiValuesFromIncompleteJson,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,header,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,function _extractRowsFromFragment,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,function _detectAndNormalizeFragment,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,traceback,function _mergeJsonStructuresGeneric,Yes +gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,dataclasses,header,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleChapterStructureFinalResult,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeStructureFinalResult,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeContentFinalResult,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subLoopingUseCases,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,json,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceAi.subResponseParsing,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,asyncio,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,base64,function _processAiResponseForSection,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,copy,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,json,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.features.aichat.serviceGeneration.renderers.registry,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonContinuation,function buildSectionPromptWithContinuation,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _extractAndMergeMultipleJsonBlocks,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processAiResponseForSection,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processSingleSection,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureFilling,typing,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,json,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,logging,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.features.aichat.serviceGeneration.renderers.registry,function generateStructure,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.shared,function generateStructure,No +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.shared.jsonContinuation,function generateStructure,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceAi.subStructureGeneration,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.__init__,(relative) .mainServiceExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,PIL,function chunk,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,io,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,json,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,docx,function _load,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,io,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,bs4,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,PIL,function extract,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,io,function extract,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,json,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,PyPDF2,function _load,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,fitz,function _load,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,io,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,io,function extract,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,pptx,function _load,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,datetime,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,io,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,openpyxl,function _load,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subRegistry,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,xml.etree.ElementTree,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerDefault,function applyMerging,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerTable,function applyMerging,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerText,function applyMerging,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subMerger,function applyMerging,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subPipeline,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subRegistry,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,asyncio,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,base64,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,json,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,function mergePartResults,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.aicore.aicoreModelSelector,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.interfaces.interfaceDbManagement,function extractContent,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.shared.debugLogger,function extractContent,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,time,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,uuid,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerDefault,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerDefault,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerText,(relative) ..subUtils,header,No +gateway.modules.features.aichat.serviceExtraction.merging.mergerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.merging.mergerText,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subMerger,(relative) .subUtils,header,Yes +gateway.modules.features.aichat.serviceExtraction.subMerger,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.subMerger,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.subMerger,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .mainServiceExtraction,function runExtraction,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .subRegistry,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .subUtils,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPipeline,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,json,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.shared.debugLogger,function buildExtractionPrompt,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerImage,function __init__,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerStructure,function __init__,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerTable,function __init__,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerText,function __init__,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .extractors.extractorBinary,function _auto_discover_extractors,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,importlib,function _auto_discover_extractors,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,logging,header,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,os,function _auto_discover_extractors,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,pathlib,function _auto_discover_extractors,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,traceback,function _auto_discover_extractors,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,traceback,function __init__,Yes +gateway.modules.features.aichat.serviceExtraction.subRegistry,typing,header,Yes +gateway.modules.features.aichat.serviceExtraction.subUtils,uuid,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,(relative) .renderers.registry,function _getFormatRenderer,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,function getAdaptiveExtractionPrompt,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.renderers.registry,function renderReport,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subContentGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subDocumentUtility,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subStructureGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,traceback,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,uuid,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelDocument,function generateCode,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.features.aichat.serviceGeneration.renderers.registry,function _getCodeRenderer,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateCodeStructure,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateSingleFileContent,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonUtils,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,time,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.codePath,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,copy,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,time,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.documentPath,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,base64,function generateImages,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,json,function generateImages,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,time,header,Yes +gateway.modules.features.aichat.serviceGeneration.paths.imagePath,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,abc,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,PIL,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,abc,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,datetime,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,function _determineFilename,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,threading,function _getAiStyles,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,importlib,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,os,function discoverRenderers,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,pathlib,function discoverRenderers,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,sys,function discoverRenderers,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.registry,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .rendererCsv,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,csv,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .rendererJson,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,xml.dom,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,xml.etree.ElementTree,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,csv,function _convertRowsToCsv,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,io,function _convertRowsToCsv,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,(relative) .rendererHtml,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,PIL,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,csv,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _setupDocumentStyles,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _createStyle,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.table,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.text,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.ns,function _renderTableFastXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _renderTableFastXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableBordersXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableRowXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _applyHorizontalBordersOnly,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackground,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackgroundFast,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.shared,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,lxml,function _renderTableFastXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _generateDocxFromJson,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderJsonTable,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderTableFastXml,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,base64,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,base64,function _replaceImageDataUris,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,html,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,html,function _replaceImageDataUris,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,re,function _replaceImageDataUris,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,re,function _extractImages,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,json,function _generateAiImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _generateAiImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _compressPromptWithAi,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,(relative) .rendererHtml,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,PIL,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,base64,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,io,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,json,function _getAiStylesWithPdfColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelAi,function _getAiStylesWithPdfColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,re,function _getAiStylesWithPdfColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,re,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.enums,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.styles,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,function _renderJsonImage,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,datetime,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addTableToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addBulletListToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addHeadingToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addParagraphToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addCodeBlockToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSlideContentWithFrames,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderTextSectionsInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSectionToTextFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addTableToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addBulletListToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addParagraphToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSlideContentWithFrames,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderTextSectionsInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSectionToTextFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addTableToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addHeadingToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addParagraphToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addCodeBlockToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderTextSectionsInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSectionToTextFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlideInFrame,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,re,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,traceback,function _addImagesToSlide,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .rendererCsv,function render,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,base64,function _addImageToExcel,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,datetime,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,dateutil,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,io,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,io,function _addImageToExcel,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,json,function _getAiStylesWithExcelColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelAi,function _getAiStylesWithExcelColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.drawing.image,function _addImageToExcel,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.styles,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.utils,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.worksheet.table,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,re,function _getAiStylesWithExcelColors,Yes +gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,asyncio,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,base64,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,base64,function _generateImageSection,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateSimpleSection,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateImageSection,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.features.aichat.serviceGeneration.subContentIntegrator,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.shared.jsonUtils,function _generateSimpleSection,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,re,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,traceback,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentGenerator,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,json,function integrateContent,Yes +gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,os,header,Yes +gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subJsonSchema,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,modules.datamodels.datamodelJson,header,Yes +gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,typing,header,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,json,header,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,json,function _createStructurePrompt,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,logging,header,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelAi,function generateStructure,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelJson,header,Yes +gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,typing,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,asyncio,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,json,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,logging,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,time,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,time,function _processCrawlResultsWithHierarchy,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,typing,header,Yes +gateway.modules.features.aichat.serviceWeb.mainServiceWeb,urllib.parse,header,Yes +gateway.modules.features.automation.mainAutomation,logging,header,Yes +gateway.modules.features.automation.mainAutomation,typing,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,(relative) .subAutomationTemplates,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,fastapi,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,fastapi,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,function get_automations,Yes +gateway.modules.features.automation.routeFeatureAutomation,json,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,logging,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.auth,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.services,function execute_automation,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.shared.attributeUtils,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.workflows.automation,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,typing,header,Yes +gateway.modules.features.automation.subAutomationTemplates,typing,header,Yes +gateway.modules.features.automation.subAutomationUtils,datetime,header,Yes +gateway.modules.features.automation.subAutomationUtils,json,header,Yes +gateway.modules.features.automation.subAutomationUtils,typing,header,Yes +gateway.modules.features.chatbot.__init__,(relative) .mainChatbot,header,Yes +gateway.modules.features.chatbot.chatbotConstants,datetime,header,Yes +gateway.modules.features.chatbot.chatbotConstants,logging,header,Yes +gateway.modules.features.chatbot.chatbotConstants,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.chatbot.chatbotConstants,re,header,Yes +gateway.modules.features.chatbot.chatbotConstants,typing,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,enum,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,modules.shared.attributeUtils,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,modules.shared.timeUtils,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,pydantic,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,typing,header,Yes +gateway.modules.features.chatbot.datamodelFeatureChatbot,uuid,header,Yes +gateway.modules.features.chatbot.eventManager,asyncio,header,Yes +gateway.modules.features.chatbot.eventManager,datetime,header,Yes +gateway.modules.features.chatbot.eventManager,logging,header,Yes +gateway.modules.features.chatbot.eventManager,typing,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,(relative) .datamodelFeatureChatbot,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,asyncio,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,datetime,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,json,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,logging,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,math,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.features.chatbot.eventManager,function createMessage,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.features.chatbot.eventManager,function createLog,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.security.rbac,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.shared.configuration,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.shared.eventManagement,function deleteAutomationDefinition,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.shared.timeUtils,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,os,function storeDebugMessageAndDocuments,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,typing,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,uuid,header,Yes +gateway.modules.features.chatbot.mainChatbot,asyncio,header,Yes +gateway.modules.features.chatbot.mainChatbot,base64,header,Yes +gateway.modules.features.chatbot.mainChatbot,json,header,Yes +gateway.modules.features.chatbot.mainChatbot,logging,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.connectors.connectorPreprocessor,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelAi,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelDocref,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.aichat.datamodelFeatureAiChat,function _emit_log_and_event,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,function _processChatbotMessage,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.eventManager,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.interfaces.interfaceRbac,function _convert_file_ids_to_document_references,Yes +gateway.modules.features.chatbot.mainChatbot,modules.services,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.shared.timeUtils,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.workflows.methods.methodAi.methodAi,header,Yes +gateway.modules.features.chatbot.mainChatbot,re,header,Yes +gateway.modules.features.chatbot.mainChatbot,typing,header,Yes +gateway.modules.features.chatbot.mainChatbot,uuid,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,(relative) .,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,(relative) .,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,(relative) .datamodelFeatureChatbot,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,(relative) .eventManager,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,asyncio,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,fastapi,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,fastapi.responses,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,json,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,logging,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,math,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.auth,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.datamodels.datamodelPagination,function get_chatbot_threads,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.shared.timeUtils,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,modules.workflows.automation,header,Yes +gateway.modules.features.chatbot.routeFeatureChatbot,typing,header,Yes +gateway.modules.features.dynamicOptions.mainDynamicOptions,logging,header,Yes +gateway.modules.features.dynamicOptions.mainDynamicOptions,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.dynamicOptions.mainDynamicOptions,typing,header,Yes +gateway.modules.features.featureRegistry,fastapi,header,Yes +gateway.modules.features.featureRegistry,glob,header,Yes +gateway.modules.features.featureRegistry,importlib,header,Yes +gateway.modules.features.featureRegistry,logging,header,Yes +gateway.modules.features.featureRegistry,os,header,Yes +gateway.modules.features.featureRegistry,typing,header,Yes +gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,modules.shared.attributeUtils,header,Yes +gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,pydantic,header,Yes +gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,typing,header,Yes +gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,uuid,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,(relative) .datamodelFeatureNeutralizer,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,asyncio,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,logging,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.datamodels.datamodelUam,function _getSharepointConnection,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.services,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.services.serviceSharepoint.mainServiceSharepoint,function processSharepointFiles,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,typing,header,Yes +gateway.modules.features.neutralizer.mainNeutralizePlayground,urllib.parse,header,Yes +gateway.modules.features.neutralizer.mainNeutralizer,logging,header,Yes +gateway.modules.features.neutralizer.mainNeutralizer,typing,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,(relative) .datamodelFeatureNeutralizer,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,(relative) .mainNeutralizePlayground,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,fastapi,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,logging,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,modules.auth,header,Yes +gateway.modules.features.neutralizer.routeFeatureNeutralizer,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,(relative) .subPatterns,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,(relative) .subProcessBinary,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,(relative) .subProcessCommon,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,(relative) .subProcessList,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,(relative) .subProcessText,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,json,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,logging,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,modules.features.neutralizer.datamodelFeatureNeutralizer,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,re,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subParseString,(relative) .subPatterns,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subParseString,re,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subParseString,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subParseString,uuid,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subPatterns,dataclasses,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subPatterns,re,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subPatterns,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessBinary,base64,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessBinary,dataclasses,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessBinary,re,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessBinary,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessCommon,dataclasses,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessCommon,pydantic,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessCommon,re,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessCommon,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,(relative) .subParseString,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,(relative) .subPatterns,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,(relative) .subPatterns,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,(relative) .subPatterns,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,dataclasses,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,io,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,json,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,pandas,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,typing,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,uuid,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,uuid,function _anonymizeTable,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,uuid,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,uuid,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,uuid,function _anonymizeXmlElement,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,xml.etree.ElementTree,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,(relative) .subParseString,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,dataclasses,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,typing,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,enum,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,modules.shared.attributeUtils,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,modules.shared.timeUtils,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,pydantic,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,typing,header,Yes +gateway.modules.features.realEstate.datamodelFeatureRealEstate,uuid,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,logging,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.security.rbac,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.shared.configuration,header,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,re,function _isUUID,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,time,function executeQuery,Yes +gateway.modules.features.realEstate.interfaceFeatureRealEstate,typing,header,Yes +gateway.modules.features.realEstate.mainRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realEstate.mainRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes +gateway.modules.features.realEstate.mainRealEstate,fastapi,header,Yes +gateway.modules.features.realEstate.mainRealEstate,json,header,Yes +gateway.modules.features.realEstate.mainRealEstate,logging,header,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,modules.services,header,Yes +gateway.modules.features.realEstate.mainRealEstate,re,function executeIntentBasedOperation,Yes +gateway.modules.features.realEstate.mainRealEstate,shapely.geometry,header,Yes +gateway.modules.features.realEstate.mainRealEstate,shapely.ops,header,Yes +gateway.modules.features.realEstate.mainRealEstate,typing,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .mainRealEstate,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,fastapi,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,json,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,logging,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,modules.auth,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,modules.shared.attributeUtils,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,requests,header,Yes +gateway.modules.features.realEstate.routeFeatureRealEstate,typing,header,Yes +gateway.modules.features.trustee.datamodelFeatureTrustee,modules.shared.attributeUtils,header,Yes +gateway.modules.features.trustee.datamodelFeatureTrustee,pydantic,header,Yes +gateway.modules.features.trustee.datamodelFeatureTrustee,typing,header,Yes +gateway.modules.features.trustee.datamodelFeatureTrustee,uuid,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,(relative) .datamodelFeatureTrustee,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,logging,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,math,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.security.rbac,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,modules.shared.configuration,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,re,function createOrganisation,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,typing,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,header,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,function createAccess,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,function createContract,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,function createDocument,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,function createPosition,Yes +gateway.modules.features.trustee.interfaceFeatureTrustee,uuid,function createPositionDocument,Yes +gateway.modules.features.trustee.mainTrustee,logging,header,Yes +gateway.modules.features.trustee.mainTrustee,typing,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,(relative) .datamodelFeatureTrustee,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,(relative) .interfaceFeatureTrustee,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,fastapi,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,fastapi,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,fastapi.responses,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,io,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,json,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,logging,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,modules.auth,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,modules.interfaces.interfaceFeatures,header,Yes +gateway.modules.features.trustee.routeFeatureTrustee,typing,header,Yes +gateway.modules.interfaces.interfaceAiObjects,asyncio,header,Yes +gateway.modules.interfaces.interfaceAiObjects,base64,header,Yes +gateway.modules.interfaces.interfaceAiObjects,dataclasses,header,Yes +gateway.modules.interfaces.interfaceAiObjects,logging,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelAi,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.features.aichat.aicore.aicoreModelSelector,header,Yes +gateway.modules.interfaces.interfaceAiObjects,time,header,Yes +gateway.modules.interfaces.interfaceAiObjects,typing,header,Yes +gateway.modules.interfaces.interfaceAiObjects,uuid,header,Yes +gateway.modules.interfaces.interfaceBootstrap,logging,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.datamodels.datamodelMembership,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.shared.configuration,header,Yes +gateway.modules.interfaces.interfaceBootstrap,modules.shared.dbMultiTenantOptimizations,function _applyDatabaseOptimizations,Yes +gateway.modules.interfaces.interfaceBootstrap,passlib.context,header,Yes +gateway.modules.interfaces.interfaceBootstrap,typing,header,Yes +gateway.modules.interfaces.interfaceDbApp,logging,header,Yes +gateway.modules.interfaces.interfaceDbApp,math,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelFeatures,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelInvitation,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelMembership,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.features.neutralizer.datamodelFeatureNeutralizer,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.interfaces.interfaceBootstrap,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.security.rbac,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.security.rootAccess,function getRootInterface,Yes +gateway.modules.interfaces.interfaceDbApp,modules.shared.configuration,header,Yes +gateway.modules.interfaces.interfaceDbApp,modules.shared.timeUtils,header,Yes +gateway.modules.interfaces.interfaceDbApp,passlib.context,header,Yes +gateway.modules.interfaces.interfaceDbApp,typing,header,Yes +gateway.modules.interfaces.interfaceDbApp,uuid,header,Yes +gateway.modules.interfaces.interfaceDbManagement,base64,header,Yes +gateway.modules.interfaces.interfaceDbManagement,hashlib,header,Yes +gateway.modules.interfaces.interfaceDbManagement,logging,header,Yes +gateway.modules.interfaces.interfaceDbManagement,math,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelFiles,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelMessaging,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelUtils,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.datamodels.datamodelVoice,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.interfaces.interfaceDbApp,function _initializeStandardPrompts,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.security.rbac,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.security.rootAccess,function _initializeStandardPrompts,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.shared.configuration,header,Yes +gateway.modules.interfaces.interfaceDbManagement,modules.shared.timeUtils,header,Yes +gateway.modules.interfaces.interfaceDbManagement,os,header,Yes +gateway.modules.interfaces.interfaceDbManagement,re,function _parse_size_string,Yes +gateway.modules.interfaces.interfaceDbManagement,typing,header,Yes +gateway.modules.interfaces.interfaceFeatures,logging,header,Yes +gateway.modules.interfaces.interfaceFeatures,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.interfaces.interfaceFeatures,modules.datamodels.datamodelFeatures,header,Yes +gateway.modules.interfaces.interfaceFeatures,modules.datamodels.datamodelMembership,function syncRolesFromTemplate,Yes +gateway.modules.interfaces.interfaceFeatures,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceFeatures,typing,header,Yes +gateway.modules.interfaces.interfaceFeatures,uuid,header,Yes +gateway.modules.interfaces.interfaceMessaging,logging,header,Yes +gateway.modules.interfaces.interfaceMessaging,modules.connectors.connectorMessagingEmail,header,Yes +gateway.modules.interfaces.interfaceMessaging,modules.connectors.connectorMessagingSms,header,Yes +gateway.modules.interfaces.interfaceMessaging,modules.datamodels.datamodelMessaging,header,Yes +gateway.modules.interfaces.interfaceMessaging,typing,header,Yes +gateway.modules.interfaces.interfaceRbac,json,header,Yes +gateway.modules.interfaces.interfaceRbac,logging,header,Yes +gateway.modules.interfaces.interfaceRbac,modules.connectors.connectorDbPostgre,function getRecordsetWithRBAC,Yes +gateway.modules.interfaces.interfaceRbac,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceRbac,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceRbac,modules.security.rbac,header,Yes +gateway.modules.interfaces.interfaceRbac,modules.security.rootAccess,header,Yes +gateway.modules.interfaces.interfaceRbac,pydantic,header,Yes +gateway.modules.interfaces.interfaceRbac,typing,header,Yes +gateway.modules.interfaces.interfaceTicketObjects,datetime,header,Yes +gateway.modules.interfaces.interfaceTicketObjects,modules.connectors.connectorTicketsClickup,function createTicketInterfaceByType,Yes +gateway.modules.interfaces.interfaceTicketObjects,modules.connectors.connectorTicketsJira,function createTicketInterfaceByType,Yes +gateway.modules.interfaces.interfaceTicketObjects,typing,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,logging,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,modules.connectors.connectorVoiceGoogle,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,modules.datamodels.datamodelVoice,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,modules.shared.timeUtils,header,Yes +gateway.modules.interfaces.interfaceVoiceObjects,typing,header,Yes +gateway.modules.routes.routeAdmin,fastapi,header,Yes +gateway.modules.routes.routeAdmin,fastapi,header,Yes +gateway.modules.routes.routeAdmin,fastapi.responses,header,Yes +gateway.modules.routes.routeAdmin,fastapi.staticfiles,header,Yes +gateway.modules.routes.routeAdmin,logging,header,Yes +gateway.modules.routes.routeAdmin,modules.auth,header,Yes +gateway.modules.routes.routeAdmin,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdmin,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeAdmin,modules.shared.configuration,header,Yes +gateway.modules.routes.routeAdmin,os,header,Yes +gateway.modules.routes.routeAdmin,pathlib,header,Yes +gateway.modules.routes.routeAdmin,typing,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,logging,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.auth,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.features.aichat.interfaceFeatureAiChat,function sync_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.interfaces.interfaceDbApp,function sync_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.services,function sync_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.shared.eventManagement,function get_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.shared.eventManagement,function remove_event,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.workflows.automation,function sync_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,typing,header,Yes +gateway.modules.routes.routeAdminFeatures,fastapi,header,Yes +gateway.modules.routes.routeAdminFeatures,fastapi,header,Yes +gateway.modules.routes.routeAdminFeatures,logging,header,Yes +gateway.modules.routes.routeAdminFeatures,modules.auth,header,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelFeatures,header,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function _getUserRoleInInstance,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function _getInstancePermissions,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function listFeatureInstanceUsers,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function addUserToFeatureInstance,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function removeUserFromFeatureInstance,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelMembership,function updateFeatureInstanceUserRoles,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function _getUserRoleInInstance,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function _getInstancePermissions,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function listFeatureInstanceUsers,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function getFeatureInstanceAvailableRoles,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function _hasMandateAdminRole,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,function listFeatureInstanceUsers,Yes +gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,function addUserToFeatureInstance,Yes +gateway.modules.routes.routeAdminFeatures,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeAdminFeatures,modules.interfaces.interfaceFeatures,header,Yes +gateway.modules.routes.routeAdminFeatures,pydantic,header,Yes +gateway.modules.routes.routeAdminFeatures,typing,header,Yes +gateway.modules.routes.routeAdminRbacExport,fastapi,header,Yes +gateway.modules.routes.routeAdminRbacExport,fastapi,header,Yes +gateway.modules.routes.routeAdminRbacExport,fastapi.responses,header,Yes +gateway.modules.routes.routeAdminRbacExport,json,header,Yes +gateway.modules.routes.routeAdminRbacExport,logging,header,Yes +gateway.modules.routes.routeAdminRbacExport,modules.auth,header,Yes +gateway.modules.routes.routeAdminRbacExport,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.routes.routeAdminRbacExport,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdminRbacExport,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeAdminRbacExport,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeAdminRbacExport,pydantic,header,Yes +gateway.modules.routes.routeAdminRbacExport,typing,header,Yes +gateway.modules.routes.routeAdminRbacRoles,fastapi,header,Yes +gateway.modules.routes.routeAdminRbacRoles,logging,header,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.auth,header,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelMembership,header,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelMembership,function listUsersWithRoles,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelUam,function listUsersWithRoles,Yes +gateway.modules.routes.routeAdminRbacRoles,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeAdminRbacRoles,typing,header,Yes +gateway.modules.routes.routeAdminRbacRules,fastapi,header,Yes +gateway.modules.routes.routeAdminRbacRules,json,header,Yes +gateway.modules.routes.routeAdminRbacRules,logging,header,Yes +gateway.modules.routes.routeAdminRbacRules,math,header,Yes +gateway.modules.routes.routeAdminRbacRules,modules.auth,header,Yes +gateway.modules.routes.routeAdminRbacRules,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeAdminRbacRules,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.routes.routeAdminRbacRules,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeAdminRbacRules,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeAdminRbacRules,typing,header,Yes +gateway.modules.routes.routeAttributes,fastapi,header,Yes +gateway.modules.routes.routeAttributes,fastapi,header,Yes +gateway.modules.routes.routeAttributes,logging,header,Yes +gateway.modules.routes.routeAttributes,modules.auth,header,Yes +gateway.modules.routes.routeAttributes,modules.shared.attributeUtils,header,Yes +gateway.modules.routes.routeDataConnections,fastapi,header,Yes +gateway.modules.routes.routeDataConnections,fastapi,header,Yes +gateway.modules.routes.routeDataConnections,json,header,Yes +gateway.modules.routes.routeDataConnections,logging,header,Yes +gateway.modules.routes.routeDataConnections,math,header,Yes +gateway.modules.routes.routeDataConnections,modules.auth,header,Yes +gateway.modules.routes.routeDataConnections,modules.auth,function get_connections,Yes +gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataConnections,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeDataConnections,modules.interfaces.interfaceDbManagement,header,Yes +gateway.modules.routes.routeDataConnections,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeDataConnections,typing,header,Yes +gateway.modules.routes.routeDataFiles,fastapi,header,Yes +gateway.modules.routes.routeDataFiles,fastapi.responses,header,Yes +gateway.modules.routes.routeDataFiles,json,header,Yes +gateway.modules.routes.routeDataFiles,logging,header,Yes +gateway.modules.routes.routeDataFiles,modules.auth,header,Yes +gateway.modules.routes.routeDataFiles,modules.datamodels.datamodelFiles,header,Yes +gateway.modules.routes.routeDataFiles,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataFiles,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataFiles,modules.interfaces.interfaceDbManagement,header,Yes +gateway.modules.routes.routeDataFiles,modules.shared.attributeUtils,header,Yes +gateway.modules.routes.routeDataFiles,typing,header,Yes +gateway.modules.routes.routeDataFiles,urllib.parse,function download_file,Yes +gateway.modules.routes.routeDataMandates,fastapi,header,Yes +gateway.modules.routes.routeDataMandates,fastapi,header,Yes +gateway.modules.routes.routeDataMandates,json,header,Yes +gateway.modules.routes.routeDataMandates,logging,header,Yes +gateway.modules.routes.routeDataMandates,modules.auth,header,Yes +gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelMembership,header,Yes +gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelMembership,function delete_mandate,Yes +gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataMandates,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeDataMandates,modules.shared.attributeUtils,header,Yes +gateway.modules.routes.routeDataMandates,modules.shared.auditLogger,header,Yes +gateway.modules.routes.routeDataMandates,pydantic,header,Yes +gateway.modules.routes.routeDataMandates,typing,header,Yes +gateway.modules.routes.routeDataPrompts,fastapi,header,Yes +gateway.modules.routes.routeDataPrompts,fastapi,header,Yes +gateway.modules.routes.routeDataPrompts,json,header,Yes +gateway.modules.routes.routeDataPrompts,logging,header,Yes +gateway.modules.routes.routeDataPrompts,modules.auth,header,Yes +gateway.modules.routes.routeDataPrompts,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataPrompts,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataPrompts,modules.datamodels.datamodelUtils,header,Yes +gateway.modules.routes.routeDataPrompts,modules.interfaces.interfaceDbManagement,header,Yes +gateway.modules.routes.routeDataPrompts,typing,header,Yes +gateway.modules.routes.routeDataUsers,fastapi,header,Yes +gateway.modules.routes.routeDataUsers,fastapi,header,Yes +gateway.modules.routes.routeDataUsers,json,header,Yes +gateway.modules.routes.routeDataUsers,logging,header,Yes +gateway.modules.routes.routeDataUsers,math,function get_users,Yes +gateway.modules.routes.routeDataUsers,modules.auth,header,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function delete_user,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function update_user,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function delete_user,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function get_user,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function reset_user_password,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function sendPasswordLink,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function create_user,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function reset_user_password,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function change_password,Yes +gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function get_users,Yes +gateway.modules.routes.routeDataUsers,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeDataUsers,modules.interfaces.interfaceDbApp,function sendPasswordLink,Yes +gateway.modules.routes.routeDataUsers,modules.services,function sendPasswordLink,Yes +gateway.modules.routes.routeDataUsers,modules.shared.auditLogger,function reset_user_password,Yes +gateway.modules.routes.routeDataUsers,modules.shared.auditLogger,function change_password,Yes +gateway.modules.routes.routeDataUsers,modules.shared.auditLogger,function sendPasswordLink,Yes +gateway.modules.routes.routeDataUsers,modules.shared.configuration,function sendPasswordLink,Yes +gateway.modules.routes.routeDataUsers,pydantic,header,Yes +gateway.modules.routes.routeDataUsers,typing,header,Yes +gateway.modules.routes.routeDataWorkflows,fastapi,header,Yes +gateway.modules.routes.routeDataWorkflows,json,header,Yes +gateway.modules.routes.routeDataWorkflows,logging,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.auth,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.services,function get_all_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.services,function get_method_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.services,function get_action_schema,Yes +gateway.modules.routes.routeDataWorkflows,modules.shared.attributeUtils,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_all_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_all_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_method_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_method_actions,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_action_schema,Yes +gateway.modules.routes.routeDataWorkflows,modules.workflows.processing.shared.methodDiscovery,function get_action_schema,Yes +gateway.modules.routes.routeDataWorkflows,typing,header,Yes +gateway.modules.routes.routeGdpr,datetime,function _timestampToIso,Yes +gateway.modules.routes.routeGdpr,fastapi,header,Yes +gateway.modules.routes.routeGdpr,fastapi,header,Yes +gateway.modules.routes.routeGdpr,fastapi.responses,header,Yes +gateway.modules.routes.routeGdpr,json,header,Yes +gateway.modules.routes.routeGdpr,logging,header,Yes +gateway.modules.routes.routeGdpr,modules.auth,header,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelFeatures,function exportUserData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelInvitation,function exportUserData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelInvitation,function deleteAccount,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function exportUserData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function exportUserData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function exportPortableData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function deleteAccount,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function deleteAccount,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelSecurity,function deleteAccount,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function deleteAccount,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function exportUserData,Yes +gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function exportPortableData,Yes +gateway.modules.routes.routeGdpr,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeGdpr,modules.shared.auditLogger,header,Yes +gateway.modules.routes.routeGdpr,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeGdpr,pydantic,header,Yes +gateway.modules.routes.routeGdpr,typing,header,Yes +gateway.modules.routes.routeInvitations,fastapi,header,Yes +gateway.modules.routes.routeInvitations,fastapi,header,Yes +gateway.modules.routes.routeInvitations,logging,header,Yes +gateway.modules.routes.routeInvitations,modules.auth,header,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelFeatures,function createInvitation,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelInvitation,header,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelRbac,function _hasMandateAdminRole,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelRbac,function _isInstanceRole,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelRbac,function createInvitation,Yes +gateway.modules.routes.routeInvitations,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeInvitations,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeInvitations,modules.security.passwordUtils,function registerAndAcceptInvitation,No +gateway.modules.routes.routeInvitations,modules.shared.configuration,function createInvitation,Yes +gateway.modules.routes.routeInvitations,modules.shared.configuration,function listInvitations,Yes +gateway.modules.routes.routeInvitations,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeInvitations,pydantic,header,Yes +gateway.modules.routes.routeInvitations,typing,header,Yes +gateway.modules.routes.routeMessaging,fastapi,header,Yes +gateway.modules.routes.routeMessaging,fastapi,header,Yes +gateway.modules.routes.routeMessaging,json,header,Yes +gateway.modules.routes.routeMessaging,logging,header,Yes +gateway.modules.routes.routeMessaging,modules.auth,header,Yes +gateway.modules.routes.routeMessaging,modules.datamodels.datamodelMessaging,header,Yes +gateway.modules.routes.routeMessaging,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.routes.routeMessaging,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.routes.routeMessaging,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeMessaging,modules.interfaces.interfaceDbApp,function _hasTriggerPermission,Yes +gateway.modules.routes.routeMessaging,modules.interfaces.interfaceDbManagement,header,Yes +gateway.modules.routes.routeMessaging,modules.services,function triggerSubscription,Yes +gateway.modules.routes.routeMessaging,typing,header,Yes +gateway.modules.routes.routeOptions,fastapi,header,Yes +gateway.modules.routes.routeOptions,logging,header,Yes +gateway.modules.routes.routeOptions,modules.auth,header,Yes +gateway.modules.routes.routeOptions,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeOptions,modules.features.dynamicOptions.mainDynamicOptions,header,Yes +gateway.modules.routes.routeOptions,modules.services,header,Yes +gateway.modules.routes.routeOptions,typing,header,Yes +gateway.modules.routes.routeSecurityAdmin,fastapi,header,Yes +gateway.modules.routes.routeSecurityAdmin,fastapi.responses,header,Yes +gateway.modules.routes.routeSecurityAdmin,logging,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.auth,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.datamodels.datamodelMembership,function revoke_tokens_by_mandate,Yes +gateway.modules.routes.routeSecurityAdmin,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeSecurityAdmin,modules.shared.configuration,header,Yes +gateway.modules.routes.routeSecurityAdmin,os,header,Yes +gateway.modules.routes.routeSecurityAdmin,typing,header,Yes +gateway.modules.routes.routeSecurityGoogle,fastapi,header,Yes +gateway.modules.routes.routeSecurityGoogle,fastapi.responses,header,Yes +gateway.modules.routes.routeSecurityGoogle,httpx,header,Yes +gateway.modules.routes.routeSecurityGoogle,jose,function auth_callback,Yes +gateway.modules.routes.routeSecurityGoogle,json,header,Yes +gateway.modules.routes.routeSecurityGoogle,logging,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.auth,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.auth,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.auth,function verify_token,Yes +gateway.modules.routes.routeSecurityGoogle,modules.auth,function refresh_token,Yes +gateway.modules.routes.routeSecurityGoogle,modules.auth,function auth_callback,Yes +gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelSecurity,function auth_callback,Yes +gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,function login,Yes +gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,function auth_callback,Yes +gateway.modules.routes.routeSecurityGoogle,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.shared.auditLogger,function logout,Yes +gateway.modules.routes.routeSecurityGoogle,modules.shared.configuration,header,Yes +gateway.modules.routes.routeSecurityGoogle,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeSecurityGoogle,requests_oauthlib,header,Yes +gateway.modules.routes.routeSecurityGoogle,typing,header,Yes +gateway.modules.routes.routeSecurityLocal,datetime,header,Yes +gateway.modules.routes.routeSecurityLocal,fastapi,header,Yes +gateway.modules.routes.routeSecurityLocal,fastapi.responses,header,Yes +gateway.modules.routes.routeSecurityLocal,fastapi.security,header,Yes +gateway.modules.routes.routeSecurityLocal,html,function _sendAuthEmail,Yes +gateway.modules.routes.routeSecurityLocal,jose,header,Yes +gateway.modules.routes.routeSecurityLocal,logging,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.auth,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.auth,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelMessaging,function _sendAuthEmail,Yes +gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,function login,Yes +gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,function register_user,Yes +gateway.modules.routes.routeSecurityLocal,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeSecurityLocal,modules.interfaces.interfaceMessaging,function _sendAuthEmail,Yes +gateway.modules.routes.routeSecurityLocal,modules.shared.auditLogger,function login,Yes +gateway.modules.routes.routeSecurityLocal,modules.shared.auditLogger,function logout,Yes +gateway.modules.routes.routeSecurityLocal,modules.shared.auditLogger,function passwordReset,Yes +gateway.modules.routes.routeSecurityLocal,modules.shared.auditLogger,function login,Yes +gateway.modules.routes.routeSecurityLocal,modules.shared.configuration,header,Yes +gateway.modules.routes.routeSecurityLocal,typing,header,Yes +gateway.modules.routes.routeSecurityLocal,uuid,header,Yes +gateway.modules.routes.routeSecurityMsft,fastapi,header,Yes +gateway.modules.routes.routeSecurityMsft,fastapi.responses,header,Yes +gateway.modules.routes.routeSecurityMsft,httpx,header,Yes +gateway.modules.routes.routeSecurityMsft,jose,function auth_callback,Yes +gateway.modules.routes.routeSecurityMsft,json,header,Yes +gateway.modules.routes.routeSecurityMsft,logging,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.auth,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.auth,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.auth,function refresh_token,Yes +gateway.modules.routes.routeSecurityMsft,modules.auth,function refresh_token,Yes +gateway.modules.routes.routeSecurityMsft,modules.auth,function auth_callback,Yes +gateway.modules.routes.routeSecurityMsft,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.shared.auditLogger,function logout,Yes +gateway.modules.routes.routeSecurityMsft,modules.shared.configuration,header,Yes +gateway.modules.routes.routeSecurityMsft,modules.shared.timeUtils,header,Yes +gateway.modules.routes.routeSecurityMsft,msal,header,Yes +gateway.modules.routes.routeSecurityMsft,typing,header,Yes +gateway.modules.routes.routeSharepoint,fastapi,header,Yes +gateway.modules.routes.routeSharepoint,logging,header,Yes +gateway.modules.routes.routeSharepoint,modules.auth,header,Yes +gateway.modules.routes.routeSharepoint,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeSharepoint,modules.interfaces.interfaceDbApp,header,Yes +gateway.modules.routes.routeSharepoint,modules.services,header,Yes +gateway.modules.routes.routeSharepoint,typing,header,Yes +gateway.modules.routes.routeVoiceGoogle,base64,header,Yes +gateway.modules.routes.routeVoiceGoogle,fastapi,header,Yes +gateway.modules.routes.routeVoiceGoogle,fastapi.responses,header,Yes +gateway.modules.routes.routeVoiceGoogle,json,header,Yes +gateway.modules.routes.routeVoiceGoogle,logging,header,Yes +gateway.modules.routes.routeVoiceGoogle,modules.auth,header,Yes +gateway.modules.routes.routeVoiceGoogle,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeVoiceGoogle,modules.interfaces.interfaceVoiceObjects,header,Yes +gateway.modules.routes.routeVoiceGoogle,typing,header,Yes +gateway.modules.security.__init__,(relative) .rbac,header,Yes +gateway.modules.security.__init__,(relative) .rbacHelpers,header,Yes +gateway.modules.security.__init__,(relative) .rootAccess,header,Yes +gateway.modules.security.rbac,logging,header,Yes +gateway.modules.security.rbac,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.security.rbac,modules.datamodels.datamodelMembership,header,Yes +gateway.modules.security.rbac,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.security.rbac,modules.datamodels.datamodelUam,header,Yes +gateway.modules.security.rbac,typing,header,Yes +gateway.modules.security.rbacCatalog,logging,header,Yes +gateway.modules.security.rbacCatalog,threading,header,Yes +gateway.modules.security.rbacCatalog,typing,header,Yes +gateway.modules.security.rbacHelpers,logging,header,Yes +gateway.modules.security.rbacHelpers,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.security.rbacHelpers,modules.datamodels.datamodelUam,header,Yes +gateway.modules.security.rbacHelpers,modules.security.rbac,header,Yes +gateway.modules.security.rbacHelpers,typing,header,Yes +gateway.modules.security.rootAccess,logging,header,Yes +gateway.modules.security.rootAccess,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.security.rootAccess,modules.datamodels.datamodelUam,header,Yes +gateway.modules.security.rootAccess,modules.interfaces.interfaceBootstrap,function _ensureBootstrap,Yes +gateway.modules.security.rootAccess,modules.shared.configuration,header,Yes +gateway.modules.services.__init__,(relative) .serviceChat.mainServiceChat,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceMessaging.mainServiceMessaging,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceSecurity.mainServiceSecurity,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceSharepoint.mainServiceSharepoint,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceTicket.mainServiceTicket,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceUtils.mainServiceUtils,function __init__,Yes +gateway.modules.services.__init__,glob,header,Yes +gateway.modules.services.__init__,importlib,header,Yes +gateway.modules.services.__init__,logging,header,Yes +gateway.modules.services.__init__,modules.datamodels.datamodelUam,header,Yes +gateway.modules.services.__init__,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.services.__init__,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.modules.services.__init__,modules.interfaces.interfaceDbManagement,function __init__,Yes +gateway.modules.services.__init__,os,header,Yes +gateway.modules.services.__init__,typing,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,json,function calculateObjectSize,Yes +gateway.modules.services.serviceChat.mainServiceChat,logging,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelDocref,function getChatDocumentsFromDocumentList,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.shared.progressLogger,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,sys,function calculateObjectSize,Yes +gateway.modules.services.serviceChat.mainServiceChat,typing,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,html,function _textToHtml,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,importlib,function _loadSubscriptionFunction,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,logging,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,modules.datamodels.datamodelMessaging,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,modules.interfaces.interfaceMessaging,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,modules.shared.timeUtils,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,re,header,Yes +gateway.modules.services.serviceMessaging.mainServiceMessaging,typing,header,Yes +gateway.modules.services.serviceMessaging.subscriptions.subSubscriptionSystemErrors,modules.datamodels.datamodelMessaging,header,Yes +gateway.modules.services.serviceMessaging.subscriptions.subSubscriptionSystemErrors,typing,header,Yes +gateway.modules.services.serviceNormalization.mainServiceNormalization,json,header,Yes +gateway.modules.services.serviceNormalization.mainServiceNormalization,os,header,Yes +gateway.modules.services.serviceNormalization.mainServiceNormalization,typing,header,Yes +gateway.modules.services.serviceSecurity.mainServiceSecurity,logging,header,Yes +gateway.modules.services.serviceSecurity.mainServiceSecurity,modules.auth,header,Yes +gateway.modules.services.serviceSecurity.mainServiceSecurity,modules.datamodels.datamodelSecurity,header,Yes +gateway.modules.services.serviceSecurity.mainServiceSecurity,typing,header,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,aiohttp,header,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,asyncio,header,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,datetime,function getFolderUsageAnalytics,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,logging,header,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,typing,header,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,urllib.parse,function findSiteByWebUrl,Yes +gateway.modules.services.serviceSharepoint.mainServiceSharepoint,urllib.parse,function getSiteByStandardPath,Yes +gateway.modules.services.serviceTicket.mainServiceTicket,logging,header,Yes +gateway.modules.services.serviceTicket.mainServiceTicket,modules.interfaces.interfaceTicketObjects,header,Yes +gateway.modules.services.serviceTicket.mainServiceTicket,typing,header,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,json,function writeDebugArtifact,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,logging,header,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.features.aichat.interfaceFeatureAiChat,function storeDebugMessageAndDocuments,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared,header,No +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.configuration,header,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function writeDebugFile,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function debugLogToFile,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function writeDebugArtifact,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.eventManagement,header,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.timeUtils,header,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,re,function sanitizePromptContent,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,typing,header,Yes +gateway.modules.shared.attributeUtils,importlib,header,Yes +gateway.modules.shared.attributeUtils,inspect,header,Yes +gateway.modules.shared.attributeUtils,logging,header,Yes +gateway.modules.shared.attributeUtils,os,header,Yes +gateway.modules.shared.attributeUtils,pydantic,header,Yes +gateway.modules.shared.attributeUtils,typing,header,Yes +gateway.modules.shared.auditLogger,datetime,header,Yes +gateway.modules.shared.auditLogger,logging,header,Yes +gateway.modules.shared.auditLogger,modules.connectors.connectorDbPostgre,function _ensureInitialized,Yes +gateway.modules.shared.auditLogger,modules.datamodels.datamodelAudit,function _ensureInitialized,Yes +gateway.modules.shared.auditLogger,modules.datamodels.datamodelAudit,function getAuditLogs,Yes +gateway.modules.shared.auditLogger,modules.datamodels.datamodelAudit,function cleanupOldEntries,Yes +gateway.modules.shared.auditLogger,modules.datamodels.datamodelAudit,function logEvent,Yes +gateway.modules.shared.auditLogger,modules.shared.configuration,header,Yes +gateway.modules.shared.auditLogger,modules.shared.eventManagement,function registerAuditLogCleanupScheduler,Yes +gateway.modules.shared.auditLogger,modules.shared.timeUtils,header,Yes +gateway.modules.shared.auditLogger,time,function cleanupOldEntries,Yes +gateway.modules.shared.auditLogger,typing,header,Yes +gateway.modules.shared.callbackRegistry,asyncio,header,Yes +gateway.modules.shared.callbackRegistry,logging,header,Yes +gateway.modules.shared.callbackRegistry,typing,header,Yes +gateway.modules.shared.configuration,base64,header,Yes +gateway.modules.shared.configuration,cryptography.fernet,header,Yes +gateway.modules.shared.configuration,cryptography.hazmat.primitives,header,Yes +gateway.modules.shared.configuration,cryptography.hazmat.primitives.kdf.pbkdf2,header,Yes +gateway.modules.shared.configuration,json,header,Yes +gateway.modules.shared.configuration,logging,header,Yes +gateway.modules.shared.configuration,modules.shared.auditLogger,function encryptValue,Yes +gateway.modules.shared.configuration,modules.shared.auditLogger,function decryptValue,Yes +gateway.modules.shared.configuration,modules.shared.auditLogger,function get,Yes +gateway.modules.shared.configuration,os,header,Yes +gateway.modules.shared.configuration,pathlib,header,Yes +gateway.modules.shared.configuration,time,header,Yes +gateway.modules.shared.configuration,typing,header,Yes +gateway.modules.shared.dbMultiTenantOptimizations,logging,header,Yes +gateway.modules.shared.dbMultiTenantOptimizations,typing,header,Yes +gateway.modules.shared.debugLogger,datetime,header,Yes +gateway.modules.shared.debugLogger,modules.shared.configuration,header,Yes +gateway.modules.shared.debugLogger,modules.shared.timeUtils,function debugLogToFile,Yes +gateway.modules.shared.debugLogger,os,header,Yes +gateway.modules.shared.debugLogger,typing,header,Yes +gateway.modules.shared.eventManagement,apscheduler.schedulers.asyncio,header,Yes +gateway.modules.shared.eventManagement,apscheduler.triggers.cron,header,Yes +gateway.modules.shared.eventManagement,apscheduler.triggers.interval,header,Yes +gateway.modules.shared.eventManagement,logging,header,Yes +gateway.modules.shared.eventManagement,typing,header,Yes +gateway.modules.shared.eventManagement,zoneinfo,header,Yes +gateway.modules.shared.frontendOptionsTypes,typing,header,Yes +gateway.modules.shared.frontendOptionsTypes,typing,header,Yes +gateway.modules.shared.frontendOptionsTypes,typing_extensions,header,Yes +gateway.modules.shared.frontendTypes,enum,header,Yes +gateway.modules.shared.frontendTypes,typing,header,Yes +gateway.modules.shared.jsonContinuation,dataclasses,header,Yes +gateway.modules.shared.jsonContinuation,enum,header,Yes +gateway.modules.shared.jsonContinuation,json,header,Yes +gateway.modules.shared.jsonContinuation,logging,header,Yes +gateway.modules.shared.jsonContinuation,modules.datamodels.datamodelAi,header,Yes +gateway.modules.shared.jsonContinuation,re,header,Yes +gateway.modules.shared.jsonContinuation,typing,header,Yes +gateway.modules.shared.jsonUtils,json,header,Yes +gateway.modules.shared.jsonUtils,logging,header,Yes +gateway.modules.shared.jsonUtils,modules.datamodels.datamodelAi,header,Yes +gateway.modules.shared.jsonUtils,modules.shared.jsonContinuation,function buildContinuationContext,Yes +gateway.modules.shared.jsonUtils,pydantic,header,Yes +gateway.modules.shared.jsonUtils,re,header,Yes +gateway.modules.shared.jsonUtils,typing,header,Yes +gateway.modules.shared.progressLogger,logging,header,Yes +gateway.modules.shared.progressLogger,time,header,Yes +gateway.modules.shared.progressLogger,typing,header,Yes +gateway.modules.shared.timeUtils,datetime,header,Yes +gateway.modules.shared.timeUtils,logging,header,Yes +gateway.modules.shared.timeUtils,time,header,Yes +gateway.modules.shared.timeUtils,typing,header,Yes +gateway.modules.workflows.automation.__init__,(relative) .mainWorkflow,header,Yes +gateway.modules.workflows.automation.mainWorkflow,(relative) .subAutomationUtils,header,Yes +gateway.modules.workflows.automation.mainWorkflow,json,header,Yes +gateway.modules.workflows.automation.mainWorkflow,logging,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.datamodels.datamodelUam,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.services,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.shared.eventManagement,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.shared.timeUtils,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.workflows.workflowManager,header,Yes +gateway.modules.workflows.automation.mainWorkflow,typing,header,Yes +gateway.modules.workflows.automation.subAutomationSchedule,logging,header,Yes +gateway.modules.workflows.automation.subAutomationSchedule,modules.services,header,Yes +gateway.modules.workflows.automation.subAutomationSchedule,modules.shared.callbackRegistry,function start,Yes +gateway.modules.workflows.automation.subAutomationSchedule,modules.workflows.automation,function start,Yes +gateway.modules.workflows.automation.subAutomationTemplates,typing,header,Yes +gateway.modules.workflows.automation.subAutomationUtils,datetime,header,Yes +gateway.modules.workflows.automation.subAutomationUtils,json,header,Yes +gateway.modules.workflows.automation.subAutomationUtils,typing,header,Yes +gateway.modules.workflows.methods.methodAi.__init__,(relative) .methodAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .convertDocument,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .generateCode,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .generateDocument,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .process,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .summarizeDocument,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .translateDocument,header,Yes +gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .webResearch,header,Yes +gateway.modules.workflows.methods.methodAi.actions.convertDocument,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.convertDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.convertDocument,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelDocref,function generateCode,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,re,function generateCode,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,time,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelDocref,function generateDocument,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,re,function generateDocument,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,time,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,json,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelDocref,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelDocref,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelWorkflow,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,time,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.translateDocument,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.translateDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.translateDocument,typing,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,re,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,time,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,typing,header,Yes +gateway.modules.workflows.methods.methodAi.helpers.csvProcessing,logging,header,Yes +gateway.modules.workflows.methods.methodAi.helpers.csvProcessing,typing,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.convertDocument,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.generateCode,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.generateDocument,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.process,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.summarizeDocument,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.translateDocument,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .actions.webResearch,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,(relative) .helpers.csvProcessing,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,datetime,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,logging,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodAi.methodAi,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodBase,datetime,header,Yes +gateway.modules.workflows.methods.methodBase,functools,header,Yes +gateway.modules.workflows.methods.methodBase,inspect,header,Yes +gateway.modules.workflows.methods.methodBase,logging,header,Yes +gateway.modules.workflows.methods.methodBase,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.workflows.methods.methodBase,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodBase,re,function _applyValidationRules,Yes +gateway.modules.workflows.methods.methodBase,re,function _generateMeaningfulFileName,Yes +gateway.modules.workflows.methods.methodBase,typing,header,Yes +gateway.modules.workflows.methods.methodChatbot.__init__,(relative) .methodChatbot,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,json,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,logging,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.connectors.connectorPreprocessor,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.datamodels.datamodelDocref,function queryDatabase,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,time,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,typing,header,Yes +gateway.modules.workflows.methods.methodChatbot.methodChatbot,(relative) .actions.queryDatabase,header,Yes +gateway.modules.workflows.methods.methodChatbot.methodChatbot,logging,header,Yes +gateway.modules.workflows.methods.methodChatbot.methodChatbot,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodChatbot.methodChatbot,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodChatbot.methodChatbot,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodContext.__init__,(relative) .methodContext,header,Yes +gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .extractContent,header,Yes +gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .getDocumentIndex,header,Yes +gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .neutralizeData,header,Yes +gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .triggerPreprocessingServer,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelDocref,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,time,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,typing,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,json,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,typing,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelDocref,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,time,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,typing,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,aiohttp,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,json,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.shared.configuration,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,typing,header,Yes +gateway.modules.workflows.methods.methodContext.helpers.documentIndex,datetime,header,Yes +gateway.modules.workflows.methods.methodContext.helpers.documentIndex,logging,header,Yes +gateway.modules.workflows.methods.methodContext.helpers.documentIndex,typing,header,Yes +gateway.modules.workflows.methods.methodContext.helpers.formatting,logging,header,Yes +gateway.modules.workflows.methods.methodContext.helpers.formatting,typing,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .actions.extractContent,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .actions.getDocumentIndex,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .actions.neutralizeData,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .actions.triggerPreprocessingServer,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .helpers.documentIndex,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,(relative) .helpers.formatting,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,logging,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodContext.methodContext,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodJira.__init__,(relative) .methodJira,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .connectJira,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .createCsvContent,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .createExcelContent,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .exportTicketsAsJson,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .importTicketsFromJson,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .mergeTicketData,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseCsvContent,header,Yes +gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseExcelContent,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.shared.configuration,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,uuid,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,base64,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,csv,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,datetime,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,io,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,pandas,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,base64,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,csv,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,datetime,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,io,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,pandas,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,io,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,pandas,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,typing,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,io,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,json,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,logging,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,pandas,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,typing,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.adfConverter,logging,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.adfConverter,typing,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.documentParsing,json,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.documentParsing,logging,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.documentParsing,modules.datamodels.datamodelDocref,header,Yes +gateway.modules.workflows.methods.methodJira.helpers.documentParsing,typing,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.connectJira,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.createCsvContent,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.createExcelContent,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.exportTicketsAsJson,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.importTicketsFromJson,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.mergeTicketData,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.parseCsvContent,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .actions.parseExcelContent,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .helpers.adfConverter,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,(relative) .helpers.documentParsing,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,logging,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodJira.methodJira,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.__init__,(relative) .methodOutlook,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .composeAndDraftEmailWithContext,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .readEmails,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .searchEmails,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .sendDraftEmail,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,base64,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,json,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,json,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,time,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,json,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,json,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.datamodels.datamodelDocref,function sendDraftEmail,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,time,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.connection,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.connection,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.connection,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.emailProcessing,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.emailProcessing,re,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.emailProcessing,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.folderManagement,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.folderManagement,requests,header,Yes +gateway.modules.workflows.methods.methodOutlook.helpers.folderManagement,typing,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .actions.composeAndDraftEmailWithContext,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .actions.readEmails,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .actions.searchEmails,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .actions.sendDraftEmail,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .helpers.connection,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .helpers.emailProcessing,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,(relative) .helpers.folderManagement,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,datetime,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodOutlook.methodOutlook,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.methods.methodSharepoint.__init__,(relative) .methodSharepoint,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .analyzeFolderUsage,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .copyFile,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .downloadFileByPath,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .findDocumentPath,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .findSiteByUrl,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .listDocuments,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .readDocuments,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .uploadDocument,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) .uploadFile,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,datetime,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,time,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.datamodels.datamodelDocref,function copyFile,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,base64,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.datamodels.datamodelDocref,function downloadFileByPath,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,os,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,time,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,urllib.parse,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,time,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,urllib.parse,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,base64,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,time,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,time,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,urllib.parse,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,base64,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,aiohttp,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,asyncio,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.connection,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.connection,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing,json,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing,modules.datamodels.datamodelDocref,function parseDocumentListForFoundDocuments,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing,modules.datamodels.datamodelDocref,function parseDocumentListForFolder,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.pathProcessing,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.pathProcessing,re,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.pathProcessing,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.siteDiscovery,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.siteDiscovery,typing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.helpers.siteDiscovery,urllib.parse,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.analyzeFolderUsage,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.copyFile,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.downloadFileByPath,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.findDocumentPath,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.findSiteByUrl,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.listDocuments,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.readDocuments,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.uploadDocument,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .actions.uploadFile,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .helpers.apiClient,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .helpers.connection,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .helpers.documentParsing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .helpers.pathProcessing,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,(relative) .helpers.siteDiscovery,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,modules.datamodels.datamodelWorkflowActions,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,modules.shared.frontendTypes,header,Yes +gateway.modules.workflows.methods.methodSharepoint.methodSharepoint,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.processing.adaptive.__init__,(relative) .contentValidator,header,Yes +gateway.modules.workflows.processing.adaptive.__init__,(relative) .learningEngine,header,Yes +gateway.modules.workflows.processing.adaptive.__init__,(relative) .progressTracker,header,Yes +gateway.modules.workflows.processing.adaptive.adaptiveLearningEngine,collections,header,Yes +gateway.modules.workflows.processing.adaptive.adaptiveLearningEngine,datetime,header,Yes +gateway.modules.workflows.processing.adaptive.adaptiveLearningEngine,logging,header,Yes +gateway.modules.workflows.processing.adaptive.adaptiveLearningEngine,typing,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,base64,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,csv,function _extractCodeFileStatistics,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,io,function _extractCodeFileStatistics,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,json,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,logging,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,re,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,typing,header,Yes +gateway.modules.workflows.processing.adaptive.contentValidator,xml.etree.ElementTree,function _extractCodeFileStatistics,Yes +gateway.modules.workflows.processing.adaptive.learningEngine,datetime,header,Yes +gateway.modules.workflows.processing.adaptive.learningEngine,logging,header,Yes +gateway.modules.workflows.processing.adaptive.learningEngine,typing,header,Yes +gateway.modules.workflows.processing.adaptive.progressTracker,datetime,header,Yes +gateway.modules.workflows.processing.adaptive.progressTracker,logging,header,Yes +gateway.modules.workflows.processing.adaptive.progressTracker,typing,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,logging,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.core.messageCreator,function _createActionCompletionMessage,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.methodDiscovery,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,time,function executeSingleAction,Yes +gateway.modules.workflows.processing.core.actionExecutor,typing,header,Yes +gateway.modules.workflows.processing.core.messageCreator,logging,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.core.messageCreator,typing,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,json,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,logging,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.features.aichat.datamodelFeatureAiChat,function generateTaskPlan,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.promptGenerationTaskplan,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,typing,header,Yes +gateway.modules.workflows.processing.core.validator,logging,header,Yes +gateway.modules.workflows.processing.core.validator,typing,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,datetime,function _createActionItem,Yes +gateway.modules.workflows.processing.modes.modeAutomation,json,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,logging,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.shared.timeUtils,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.modes.modeBase,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,typing,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,uuid,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,uuid,function _createActionItem,Yes +gateway.modules.workflows.processing.modes.modeBase,abc,header,Yes +gateway.modules.workflows.processing.modes.modeBase,logging,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.actionExecutor,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.messageCreator,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.taskPlanner,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.validator,header,Yes +gateway.modules.workflows.processing.modes.modeBase,typing,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,datetime,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,json,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,logging,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _planSelect,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _planSelect,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _planSelect,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelWorkflow,function _planSelect,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelWorkflow,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _planSelect,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.timeUtils,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.adaptive,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.adaptive.adaptiveLearningEngine,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.modes.modeBase,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.executionState,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.methodDiscovery,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.methodDiscovery,function _actExecute,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.placeholderFactory,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.promptGenerationActionsDynamic,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,re,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,time,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,typing,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes +gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes +gateway.modules.workflows.processing.shared.executionState,logging,header,Yes +gateway.modules.workflows.processing.shared.executionState,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.executionState,typing,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,importlib,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,inspect,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,logging,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,modules.workflows.methods.methodBase,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,pkgutil,header,Yes +gateway.modules.workflows.processing.shared.methodDiscovery,typing,header,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,json,header,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,logging,header,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.interfaceFeatureAiChat,function extractLatestRefinementFeedback,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.interfaces.interfaceDbApp,function extractLatestRefinementFeedback,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.workflows.processing.shared.methodDiscovery,header,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,typing,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,json,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.methodDiscovery,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.placeholderFactory,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,typing,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,logging,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.workflows.processing.shared.placeholderFactory,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,typing,header,Yes +gateway.modules.workflows.processing.shared.stateTools,logging,header,Yes +gateway.modules.workflows.processing.shared.stateTools,typing,header,Yes +gateway.modules.workflows.processing.workflowProcessor,json,header,Yes +gateway.modules.workflows.processing.workflowProcessor,logging,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,function fastPathExecute,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,function fastPathExecute,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,function persistTaskResult,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,function initialUnderstanding,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeAutomation,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeBase,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeDynamic,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.shared.stateTools,function persistTaskResult,Yes +gateway.modules.workflows.processing.workflowProcessor,time,function generateTaskPlan,Yes +gateway.modules.workflows.processing.workflowProcessor,time,function executeTask,Yes +gateway.modules.workflows.processing.workflowProcessor,traceback,function fastPathExecute,Yes +gateway.modules.workflows.processing.workflowProcessor,typing,header,Yes +gateway.modules.workflows.workflowManager,asyncio,header,Yes +gateway.modules.workflows.workflowManager,json,header,Yes +gateway.modules.workflows.workflowManager,logging,header,Yes +gateway.modules.workflows.workflowManager,modules.datamodels.datamodelWorkflow,function _executeTasks,Yes +gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _executeTasks,Yes +gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes +gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes +gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes +gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes +gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.placeholderFactory,function _checkIfHistoryAvailable,Yes +gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.workflows.workflowManager,modules.workflows.processing.workflowProcessor,header,Yes +gateway.modules.workflows.workflowManager,typing,header,Yes +gateway.modules.workflows.workflowManager,uuid,header,Yes +gateway.scripts.script_analyze_imports,ast,header,Yes +gateway.scripts.script_analyze_imports,csv,header,Yes +gateway.scripts.script_analyze_imports,os,header,Yes +gateway.scripts.script_analyze_imports,pathlib,header,Yes +gateway.scripts.script_analyze_imports,sys,header,Yes +gateway.scripts.script_analyze_imports,typing,header,Yes +gateway.scripts.script_db_adapt_to_models,argparse,header,Yes +gateway.scripts.script_db_adapt_to_models,ast,function _parsePydanticModels,Yes +gateway.scripts.script_db_adapt_to_models,ast,function _extractType,Yes +gateway.scripts.script_db_adapt_to_models,json,header,Yes +gateway.scripts.script_db_adapt_to_models,logging,header,Yes +gateway.scripts.script_db_adapt_to_models,modules.shared.configuration,header,Yes +gateway.scripts.script_db_adapt_to_models,os,header,Yes +gateway.scripts.script_db_adapt_to_models,pathlib,header,Yes +gateway.scripts.script_db_adapt_to_models,psycopg2,header,Yes +gateway.scripts.script_db_adapt_to_models,psycopg2.extras,header,Yes +gateway.scripts.script_db_adapt_to_models,sys,header,Yes +gateway.scripts.script_db_adapt_to_models,typing,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,argparse,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,dotenv,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,logging,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,modules.datamodels.datamodelRbac,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,modules.security.rootAccess,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,os,header,Yes +gateway.scripts.script_db_cleanup_duplicate_roles,sys,header,Yes +gateway.scripts.script_db_export_migration,argparse,header,Yes +gateway.scripts.script_db_export_migration,datetime,header,Yes +gateway.scripts.script_db_export_migration,json,header,Yes +gateway.scripts.script_db_export_migration,logging,header,Yes +gateway.scripts.script_db_export_migration,modules.shared.configuration,header,Yes +gateway.scripts.script_db_export_migration,os,header,Yes +gateway.scripts.script_db_export_migration,pathlib,header,Yes +gateway.scripts.script_db_export_migration,psycopg2,header,Yes +gateway.scripts.script_db_export_migration,psycopg2.extras,header,Yes +gateway.scripts.script_db_export_migration,sys,header,Yes +gateway.scripts.script_db_export_migration,typing,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,argparse,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,datetime,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,modules.shared.configuration,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,pathlib,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,shutil,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,sys,header,Yes +gateway.scripts.script_security_encrypt_all_env_files,typing,header,Yes +gateway.scripts.script_security_encrypt_config_value,argparse,header,Yes +gateway.scripts.script_security_encrypt_config_value,datetime,header,Yes +gateway.scripts.script_security_encrypt_config_value,json,header,Yes +gateway.scripts.script_security_encrypt_config_value,modules.shared.configuration,header,Yes +gateway.scripts.script_security_encrypt_config_value,modules.shared.configuration,function main,Yes +gateway.scripts.script_security_encrypt_config_value,os,header,Yes +gateway.scripts.script_security_encrypt_config_value,pathlib,header,Yes +gateway.scripts.script_security_encrypt_config_value,shutil,header,Yes +gateway.scripts.script_security_encrypt_config_value,sys,header,Yes +gateway.scripts.script_security_generate_master_keys,argparse,header,Yes +gateway.scripts.script_security_generate_master_keys,base64,header,Yes +gateway.scripts.script_security_generate_master_keys,os,header,Yes +gateway.scripts.script_security_generate_master_keys,pathlib,header,Yes +gateway.scripts.script_security_generate_master_keys,secrets,header,Yes +gateway.scripts.script_security_generate_master_keys,sys,header,Yes +gateway.scripts.script_stats_durations_from_log,argparse,header,Yes +gateway.scripts.script_stats_durations_from_log,csv,header,Yes +gateway.scripts.script_stats_durations_from_log,datetime,header,Yes +gateway.scripts.script_stats_durations_from_log,re,header,Yes +gateway.scripts.script_stats_durations_from_log,typing,header,Yes +gateway.scripts.script_stats_get_codelines,argparse,header,Yes +gateway.scripts.script_stats_get_codelines,os,header,Yes +gateway.scripts.script_stats_get_codelines,pathlib,header,Yes +gateway.scripts.script_stats_get_codelines,typing,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,ast,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,logging,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,os,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,pathlib,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,re,header,Yes +gateway.scripts.script_stats_showUnusedFunctions,typing,header,Yes +gateway.tests.conftest,os,header,Yes +gateway.tests.conftest,pathlib,header,Yes +gateway.tests.conftest,sys,header,Yes +gateway.tests.functional.test01_ai_model_selection,asyncio,header,Yes +gateway.tests.functional.test01_ai_model_selection,base64,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.aicore.aicoreModelSelector,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.serviceAi.mainServiceAi,function initialize,Yes +gateway.tests.functional.test01_ai_model_selection,modules.interfaces.interfaceAiObjects,function initialize,Yes +gateway.tests.functional.test01_ai_model_selection,modules.services,header,Yes +gateway.tests.functional.test01_ai_model_selection,os,header,Yes +gateway.tests.functional.test01_ai_model_selection,sys,header,Yes +gateway.tests.functional.test02_ai_models,asyncio,header,Yes +gateway.tests.functional.test02_ai_models,base64,header,Yes +gateway.tests.functional.test02_ai_models,base64,function _createTestImage,Yes +gateway.tests.functional.test02_ai_models,base64,function _saveImageResponse,Yes +gateway.tests.functional.test02_ai_models,base64,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,collections,function printTestSummary,Yes +gateway.tests.functional.test02_ai_models,datetime,header,Yes +gateway.tests.functional.test02_ai_models,json,header,Yes +gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,logging,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function _getTestPromptForOperation,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function getAllAvailableModels,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function testModel,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function getAllAvailableModels,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicorePluginPerplexity,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicorePluginTavily,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.datamodelFeatureAiChat,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.serviceAi.mainServiceAi,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.features.aichat.serviceExtraction.mainServiceExtraction,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.services,header,Yes +gateway.tests.functional.test02_ai_models,modules.shared.configuration,function _testTavilyDirect,Yes +gateway.tests.functional.test02_ai_models,os,header,Yes +gateway.tests.functional.test02_ai_models,sys,header,Yes +gateway.tests.functional.test02_ai_models,tavily,function _testTavilyDirect,Yes +gateway.tests.functional.test02_ai_models,typing,header,Yes +gateway.tests.functional.test02_ai_models,uuid,function initialize,Yes +gateway.tests.functional.test03_ai_operations,asyncio,header,Yes +gateway.tests.functional.test03_ai_operations,datetime,header,Yes +gateway.tests.functional.test03_ai_operations,json,function printSummary,Yes +gateway.tests.functional.test03_ai_operations,json,function testOperation,Yes +gateway.tests.functional.test03_ai_operations,logging,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function testOperation,Yes +gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test03_ai_operations,modules.services,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.workflows.methods.methodAi,function initialize,Yes +gateway.tests.functional.test03_ai_operations,os,header,Yes +gateway.tests.functional.test03_ai_operations,sys,header,Yes +gateway.tests.functional.test03_ai_operations,time,function initialize,Yes +gateway.tests.functional.test03_ai_operations,time,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,time,function testOperation,Yes +gateway.tests.functional.test03_ai_operations,typing,header,Yes +gateway.tests.functional.test03_ai_operations,uuid,function initialize,Yes +gateway.tests.functional.test03_ai_operations,uuid,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,uuid,function testOperation,Yes +gateway.tests.functional.test04_ai_behavior,asyncio,header,Yes +gateway.tests.functional.test04_ai_behavior,glob,function _getLatestDebugResponse,Yes +gateway.tests.functional.test04_ai_behavior,json,header,Yes +gateway.tests.functional.test04_ai_behavior,logging,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.functional.test04_ai_behavior,modules.features.aichat.datamodelFeatureAiChat,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,modules.features.aichat.interfaceFeatureAiChat,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test04_ai_behavior,modules.services,header,Yes +gateway.tests.functional.test04_ai_behavior,os,header,Yes +gateway.tests.functional.test04_ai_behavior,sys,header,Yes +gateway.tests.functional.test04_ai_behavior,time,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,traceback,function testPromptBehavior,Yes +gateway.tests.functional.test04_ai_behavior,typing,header,Yes +gateway.tests.functional.test04_ai_behavior,uuid,function initialize,Yes +gateway.tests.functional.test05_workflow_with_documents,asyncio,header,Yes +gateway.tests.functional.test05_workflow_with_documents,json,header,Yes +gateway.tests.functional.test05_workflow_with_documents,logging,function initialize,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.services,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.workflows.automation,header,Yes +gateway.tests.functional.test05_workflow_with_documents,os,header,Yes +gateway.tests.functional.test05_workflow_with_documents,sys,header,Yes +gateway.tests.functional.test05_workflow_with_documents,time,header,Yes +gateway.tests.functional.test05_workflow_with_documents,traceback,function runTest,Yes +gateway.tests.functional.test05_workflow_with_documents,typing,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,asyncio,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,json,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,logging,function initialize,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.services,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.workflows.automation,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,os,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,sys,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,time,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,traceback,function testSimplePrompt,Yes +gateway.tests.functional.test06_workflow_prompt_variations,traceback,function testMergeDocumentsToWord,Yes +gateway.tests.functional.test06_workflow_prompt_variations,traceback,function testStructuredDataToExcel,Yes +gateway.tests.functional.test06_workflow_prompt_variations,traceback,function runAllTests,Yes +gateway.tests.functional.test06_workflow_prompt_variations,typing,header,Yes +gateway.tests.functional.test07_json_merge,json,header,Yes +gateway.tests.functional.test07_json_merge,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test07_json_merge,modules.shared.jsonUtils,header,Yes +gateway.tests.functional.test07_json_merge,os,header,Yes +gateway.tests.functional.test07_json_merge,sys,header,Yes +gateway.tests.functional.test07_json_merge,traceback,header,Yes +gateway.tests.functional.test08_json_finalization,json,header,Yes +gateway.tests.functional.test08_json_finalization,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test08_json_finalization,modules.shared.jsonUtils,header,Yes +gateway.tests.functional.test08_json_finalization,os,header,Yes +gateway.tests.functional.test08_json_finalization,sys,header,Yes +gateway.tests.functional.test08_json_finalization,traceback,function testEndToEndFinalizationWithCorruption,Yes +gateway.tests.functional.test08_json_finalization,traceback,header,Yes +gateway.tests.functional.test09_document_generation_formats,asyncio,header,Yes +gateway.tests.functional.test09_document_generation_formats,base64,header,Yes +gateway.tests.functional.test09_document_generation_formats,json,header,Yes +gateway.tests.functional.test09_document_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test09_document_generation_formats,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test09_document_generation_formats,modules.services,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.shared.configuration,function initialize,Yes +gateway.tests.functional.test09_document_generation_formats,modules.workflows.automation,header,Yes +gateway.tests.functional.test09_document_generation_formats,os,header,Yes +gateway.tests.functional.test09_document_generation_formats,sys,header,Yes +gateway.tests.functional.test09_document_generation_formats,time,header,Yes +gateway.tests.functional.test09_document_generation_formats,traceback,function uploadPdfFile,Yes +gateway.tests.functional.test09_document_generation_formats,traceback,function runTest,Yes +gateway.tests.functional.test09_document_generation_formats,traceback,function testRefactoringFeatures,Yes +gateway.tests.functional.test09_document_generation_formats,traceback,function testAllFormats,Yes +gateway.tests.functional.test09_document_generation_formats,typing,header,Yes +gateway.tests.functional.test10_document_generation_formats,asyncio,header,Yes +gateway.tests.functional.test10_document_generation_formats,base64,header,Yes +gateway.tests.functional.test10_document_generation_formats,json,header,Yes +gateway.tests.functional.test10_document_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test10_document_generation_formats,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test10_document_generation_formats,modules.services,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.shared.configuration,function initialize,Yes +gateway.tests.functional.test10_document_generation_formats,modules.workflows.automation,header,Yes +gateway.tests.functional.test10_document_generation_formats,os,header,Yes +gateway.tests.functional.test10_document_generation_formats,sys,header,Yes +gateway.tests.functional.test10_document_generation_formats,time,header,Yes +gateway.tests.functional.test10_document_generation_formats,traceback,function uploadPdfFile,Yes +gateway.tests.functional.test10_document_generation_formats,traceback,function runTest,Yes +gateway.tests.functional.test10_document_generation_formats,traceback,function testAllFormats,Yes +gateway.tests.functional.test10_document_generation_formats,typing,header,Yes +gateway.tests.functional.test11_code_generation_formats,asyncio,header,Yes +gateway.tests.functional.test11_code_generation_formats,csv,header,Yes +gateway.tests.functional.test11_code_generation_formats,io,header,Yes +gateway.tests.functional.test11_code_generation_formats,json,header,Yes +gateway.tests.functional.test11_code_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test11_code_generation_formats,modules.datamodels.datamodelUam,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test11_code_generation_formats,modules.services,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.shared.configuration,function initialize,Yes +gateway.tests.functional.test11_code_generation_formats,modules.workflows.automation,header,Yes +gateway.tests.functional.test11_code_generation_formats,os,header,Yes +gateway.tests.functional.test11_code_generation_formats,sys,header,Yes +gateway.tests.functional.test11_code_generation_formats,time,header,Yes +gateway.tests.functional.test11_code_generation_formats,traceback,function runTest,Yes +gateway.tests.functional.test11_code_generation_formats,traceback,function testAllFormats,Yes +gateway.tests.functional.test11_code_generation_formats,typing,header,Yes +gateway.tests.functional.test11_code_generation_formats,xml.etree.ElementTree,header,Yes +gateway.tests.functional.test12_json_split_merge,asyncio,header,Yes +gateway.tests.functional.test12_json_split_merge,json,header,Yes +gateway.tests.functional.test12_json_split_merge,modules.features.aichat.serviceAi.subJsonMerger,header,Yes +gateway.tests.functional.test12_json_split_merge,modules.shared.jsonContinuation,header,Yes +gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function _loadTableJsonExample,Yes +gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function testJsonSplitMerge,Yes +gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function normalizeJson,Yes +gateway.tests.functional.test12_json_split_merge,os,header,Yes +gateway.tests.functional.test12_json_split_merge,random,header,Yes +gateway.tests.functional.test12_json_split_merge,random,function testJsonSplitMerge,Yes +gateway.tests.functional.test12_json_split_merge,sys,header,Yes +gateway.tests.functional.test12_json_split_merge,time,header,Yes +gateway.tests.functional.test12_json_split_merge,traceback,function runTest,Yes +gateway.tests.functional.test12_json_split_merge,traceback,function testAllJsonFiles,Yes +gateway.tests.functional.test12_json_split_merge,typing,header,Yes +gateway.tests.functional.test13_json_completion_cuts,asyncio,header,Yes +gateway.tests.functional.test13_json_completion_cuts,json,header,Yes +gateway.tests.functional.test13_json_completion_cuts,modules.shared.jsonContinuation,header,Yes +gateway.tests.functional.test13_json_completion_cuts,os,header,Yes +gateway.tests.functional.test13_json_completion_cuts,sys,header,Yes +gateway.tests.functional.test13_json_completion_cuts,traceback,function runTest,Yes +gateway.tests.functional.test13_json_completion_cuts,typing,header,Yes +gateway.tests.functional.test14_json_continuation_context,asyncio,header,Yes +gateway.tests.functional.test14_json_continuation_context,json,header,Yes +gateway.tests.functional.test14_json_continuation_context,modules.shared.jsonContinuation,header,Yes +gateway.tests.functional.test14_json_continuation_context,os,header,Yes +gateway.tests.functional.test14_json_continuation_context,sys,header,Yes +gateway.tests.functional.test14_json_continuation_context,traceback,function testSpecificCutJson,Yes +gateway.tests.functional.test14_json_continuation_context,traceback,function runTest,Yes +gateway.tests.functional.test14_json_continuation_context,typing,header,Yes +gateway.tests.functional.test_kpi_full,json,header,Yes +gateway.tests.functional.test_kpi_full,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test_kpi_full,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test_kpi_full,modules.shared.jsonUtils,header,Yes +gateway.tests.functional.test_kpi_full,os,header,Yes +gateway.tests.functional.test_kpi_full,pytest,header,Yes +gateway.tests.functional.test_kpi_full,sys,header,Yes +gateway.tests.functional.test_kpi_incomplete,json,header,Yes +gateway.tests.functional.test_kpi_incomplete,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test_kpi_incomplete,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test_kpi_incomplete,modules.shared.jsonUtils,header,Yes +gateway.tests.functional.test_kpi_incomplete,os,header,Yes +gateway.tests.functional.test_kpi_incomplete,pytest,header,Yes +gateway.tests.functional.test_kpi_incomplete,sys,header,Yes +gateway.tests.functional.test_kpi_incomplete,traceback,header,Yes +gateway.tests.functional.test_kpi_path,json,header,Yes +gateway.tests.functional.test_kpi_path,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test_kpi_path,os,header,Yes +gateway.tests.functional.test_kpi_path,sys,header,Yes +gateway.tests.functional.test_kpi_path,traceback,header,Yes +gateway.tests.integration.options.test_options_api,app,function app,Yes +gateway.tests.integration.options.test_options_api,fastapi.testclient,header,Yes +gateway.tests.integration.options.test_options_api,modules.datamodels.datamodelUam,header,Yes +gateway.tests.integration.options.test_options_api,modules.interfaces.interfaceDbApp,header,Yes +gateway.tests.integration.options.test_options_api,pytest,header,Yes +gateway.tests.integration.options.test_options_api,secrets,header,Yes +gateway.tests.integration.rbac.test_rbac_database,modules.connectors.connectorDbPostgre,header,Yes +gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUam,header,Yes +gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUam,function testBuildRbacWhereClauseUserConnectionTable,Yes +gateway.tests.integration.rbac.test_rbac_database,modules.shared.configuration,header,Yes +gateway.tests.integration.rbac.test_rbac_database,pytest,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelDocref,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,function test_extractContentParameters_structure,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.shared.jsonUtils,function test_parseJsonWithModel_with_code_fences,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.shared.jsonUtils,function test_parseJsonWithModel_with_extra_text,Yes +gateway.tests.integration.workflows.test_workflow_execution,pytest,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,unittest.mock,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,uuid,header,Yes +gateway.tests.unit.datamodels.test_docref,modules.datamodels.datamodelDocref,header,Yes +gateway.tests.unit.datamodels.test_docref,pytest,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,json,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelAi,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelDocref,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelExtraction,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,pytest,header,Yes +gateway.tests.unit.datamodels.test_workflow_models,typing,header,Yes +gateway.tests.unit.options.test_frontend_options_types,modules.shared.frontendOptionsTypes,header,Yes +gateway.tests.unit.options.test_frontend_options_types,pytest,header,Yes +gateway.tests.unit.options.test_main_options,modules.datamodels.datamodelUam,header,Yes +gateway.tests.unit.options.test_main_options,modules.features.dynamicOptions.mainDynamicOptions,header,Yes +gateway.tests.unit.options.test_main_options,pytest,header,Yes +gateway.tests.unit.options.test_main_options,unittest.mock,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelRbac,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelUam,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelUam,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,modules.interfaces.interfaceBootstrap,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,pytest,header,Yes +gateway.tests.unit.rbac.test_rbac_bootstrap,unittest.mock,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,modules.connectors.connectorDbPostgre,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,modules.datamodels.datamodelRbac,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,modules.datamodels.datamodelUam,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,modules.security.rbac,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,pytest,header,Yes +gateway.tests.unit.rbac.test_rbac_permissions,unittest.mock,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,json,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,modules.datamodels.datamodelExtraction,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,modules.features.aichat.serviceExtraction.mainServiceExtraction,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,os,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,sys,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,traceback,function main,Yes +gateway.tests.unit.utils.test_json_utils,json,header,Yes +gateway.tests.unit.utils.test_json_utils,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.unit.utils.test_json_utils,modules.shared.jsonUtils,header,Yes +gateway.tests.unit.utils.test_json_utils,pytest,header,Yes +gateway.tests.unit.workflows.test_state_management,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.unit.workflows.test_state_management,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.unit.workflows.test_state_management,pytest,header,Yes +gateway.tests.unit.workflows.test_state_management,uuid,header,Yes +gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelDocref,header,Yes +gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelWorkflow,header,Yes +gateway.tests.validation.test_architecture_validation,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.validation.test_architecture_validation,modules.shared.jsonUtils,header,Yes +gateway.tests.validation.test_architecture_validation,os,header,Yes +gateway.tests.validation.test_architecture_validation,pytest,header,Yes +gateway.tests.validation.test_architecture_validation,sys,header,Yes diff --git a/scripts/script_analyze_imports.py b/scripts/script_analyze_imports.py new file mode 100644 index 00000000..b6bf9632 --- /dev/null +++ b/scripts/script_analyze_imports.py @@ -0,0 +1,196 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Analyze all imports in the gateway codebase and generate a CSV report. +Helps identify which imports need to be cleaned up after refactoring. +""" + +import os +import ast +import csv +import sys +from pathlib import Path +from typing import List, Tuple, Optional + +# Gateway root directory +GATEWAY_ROOT = Path(__file__).parent.parent +OUTPUT_FILE = GATEWAY_ROOT / "scripts" / "import_analysis.csv" + + +def getModuleName(filePath: Path) -> str: + """Convert file path to module name format.""" + relPath = filePath.relative_to(GATEWAY_ROOT.parent) + # Remove .py extension and convert to module format + modulePath = str(relPath).replace(os.sep, ".").replace("/", ".") + if modulePath.endswith(".py"): + modulePath = modulePath[:-3] + return modulePath + + +def getImportedModuleName(importNode: ast.AST) -> List[str]: + """Extract imported module names from import node.""" + modules = [] + + if isinstance(importNode, ast.Import): + for alias in importNode.names: + modules.append(alias.name) + elif isinstance(importNode, ast.ImportFrom): + if importNode.module: + # For relative imports, we'll mark them specially + if importNode.level > 0: + modules.append(f"{'.' * importNode.level}{importNode.module or ''}") + else: + modules.append(importNode.module) + elif importNode.level > 0: + # Pure relative import like "from . import x" + modules.append("." * importNode.level) + + return modules + + +def findEnclosingFunction(node: ast.AST, tree: ast.Module) -> Optional[str]: + """Find the function name that contains this import, if any.""" + for item in ast.walk(tree): + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + for child in ast.walk(item): + if child is node: + return item.name + return None + + +def checkModuleExists(moduleName: str, currentFile: Path) -> bool: + """Check if an imported module exists.""" + if moduleName.startswith("."): + # Relative import - check relative to current file's directory + currentDir = currentFile.parent + relativePath = moduleName.lstrip(".") + + if not relativePath: + return True # "from . import x" - current package + + # Convert module path to file path + parts = relativePath.split(".") + checkPath = currentDir + for part in parts: + checkPath = checkPath / part + + # Check if it's a package or module + if checkPath.is_dir() and (checkPath / "__init__.py").exists(): + return True + if (checkPath.parent / f"{checkPath.name}.py").exists(): + return True + if checkPath.with_suffix(".py").exists(): + return True + + return False + + # Absolute import + if moduleName.startswith("modules."): + # Internal module - check in gateway + parts = moduleName.split(".") + checkPath = GATEWAY_ROOT + for part in parts: + checkPath = checkPath / part + + # Check if it's a package or module + if checkPath.is_dir() and (checkPath / "__init__.py").exists(): + return True + if checkPath.with_suffix(".py").exists(): + return True + + return False + + # External module - assume it exists (can't easily verify without importing) + return True + + +def analyzeFile(filePath: Path) -> List[Tuple[str, str, str, str]]: + """Analyze imports in a single file.""" + results = [] + moduleName = getModuleName(filePath) + + try: + with open(filePath, "r", encoding="utf-8") as f: + content = f.read() + + tree = ast.parse(content, filename=str(filePath)) + except (SyntaxError, UnicodeDecodeError) as e: + print(f"Error parsing {filePath}: {e}") + return results + + # Find all imports and their positions + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + importedModules = getImportedModuleName(node) + + # Determine position + position = "header" + + # Check if import is inside a function by examining parent nodes + # We need to traverse the tree structure + for item in ast.walk(tree): + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + # Check if this import node is within the function's body + for bodyNode in ast.walk(item): + if bodyNode is node: + position = f"function {item.name}" + break + if position != "header": + break + + for importedModule in importedModules: + # Check if module exists + exists = checkModuleExists(importedModule, filePath) + validStr = "Yes" if exists else "No" + + # Format imported module name + if importedModule.startswith("."): + # Make relative import absolute for clarity + importedModuleDisplay = f"(relative) {importedModule}" + else: + importedModuleDisplay = importedModule + + results.append((moduleName, importedModuleDisplay, position, validStr)) + + return results + + +def main(): + """Main function to analyze all imports.""" + allResults = [] + + # Find all Python files in gateway + for root, dirs, files in os.walk(GATEWAY_ROOT): + # Skip __pycache__ directories + dirs[:] = [d for d in dirs if d != "__pycache__"] + + for file in files: + if file.endswith(".py"): + filePath = Path(root) / file + results = analyzeFile(filePath) + allResults.extend(results) + + # Sort results + allResults.sort(key=lambda x: (x[0], x[1])) + + # Write CSV + with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["module_name", "imported_module_name", "position", "import_valid"]) + writer.writerows(allResults) + + print(f"Analysis complete. Found {len(allResults)} imports.") + print(f"Output written to: {OUTPUT_FILE}") + + # Print summary of invalid imports + invalidImports = [r for r in allResults if r[3] == "No"] + if invalidImports: + print(f"\nFound {len(invalidImports)} potentially invalid imports:") + for moduleName, importedModule, position, _ in invalidImports[:20]: + print(f" {moduleName} -> {importedModule} ({position})") + if len(invalidImports) > 20: + print(f" ... and {len(invalidImports) - 20} more") + + +if __name__ == "__main__": + main() diff --git a/tests/functional/test01_ai_model_selection.py b/tests/functional/test01_ai_model_selection.py index b06e9c64..0cf36f47 100644 --- a/tests/functional/test01_ai_model_selection.py +++ b/tests/functional/test01_ai_model_selection.py @@ -29,8 +29,8 @@ from modules.datamodels.datamodelAi import ( ProcessingModeEnum, ) from modules.datamodels.datamodelUam import User -from modules.aicore.aicoreModelRegistry import modelRegistry -from modules.aicore.aicoreModelSelector import modelSelector +from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.features.aichat.aicore.aicoreModelSelector import modelSelector class ModelSelectionTester: @@ -46,7 +46,7 @@ class ModelSelectionTester: self.services = getServices(testUser, None) async def initialize(self) -> None: - from modules.services.serviceAi.mainServiceAi import AiService + from modules.features.aichat.serviceAi.mainServiceAi import AiService from modules.interfaces.interfaceAiObjects import AiObjects self.services.ai = await AiService.create(self.services) diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 12a374f8..00953f3e 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -68,24 +68,24 @@ class AIModelsTester: logging.getLogger().setLevel(logging.DEBUG) # Initialize the model registry with all connectors - from modules.aicore.aicoreModelRegistry import modelRegistry - from modules.aicore.aicorePluginTavily import AiTavily - from modules.aicore.aicorePluginPerplexity import AiPerplexity + from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.features.aichat.aicore.aicorePluginTavily import AiTavily + from modules.features.aichat.aicore.aicorePluginPerplexity import AiPerplexity # Note: We don't need to register web connectors for IMAGE_ANALYSE testing # modelRegistry.registerConnector(AiTavily()) # modelRegistry.registerConnector(AiPerplexity()) # The AI service needs to be recreated with proper initialization - from modules.services.serviceAi.mainServiceAi import AiService + from modules.features.aichat.serviceAi.mainServiceAi import AiService self.services.ai = await AiService.create(self.services) # Also initialize extraction service for image processing - from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService + from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( @@ -311,7 +311,7 @@ class AIModelsTester: print(f"{'='*60}") # Get model from registry - from modules.aicore.aicoreModelRegistry import modelRegistry + from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry model = modelRegistry.getModel(modelName) if not model: @@ -693,7 +693,7 @@ Width: {crawlWidth} def getAllAvailableModels(self) -> List[Dict[str, Any]]: """Get all available models with their supported operation types.""" - from modules.aicore.aicoreModelRegistry import modelRegistry + from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry from modules.datamodels.datamodelAi import OperationTypeEnum # Get all models from registry diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index f807fe24..5875c2bb 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -94,7 +94,7 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.interfaces.interfaceDbChat as interfaceDbChat + import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) # Import and initialize services @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.datamodels.datamodelChat import ChatDocument + from modules.features.aichat.datamodelFeatureAiChat import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.datamodels.datamodelChat import ChatMessage + from modules.features.aichat.datamodelFeatureAiChat import ChatMessage import time testMessage = ChatMessage( @@ -201,7 +201,7 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.interfaces.interfaceDbChat as interfaceDbChat + import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,7 +283,7 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.interfaces.interfaceDbChat as interfaceDbChat + import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 3e46bc0c..6b28439a 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,10 +42,10 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.interfaces.interfaceDbChat as interfaceDbChat + import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat currentTimestamp = time.time() diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 8850ae2b..7e6347c2 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,10 +20,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User -from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.workflows.automation import chatStart +import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class WorkflowWithDocumentsTester: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 106e2999..ada63dea 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,10 +22,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User -from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.workflows.automation import chatStart +import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class WorkflowPromptVariationsTester: diff --git a/tests/functional/test07_json_merge.py b/tests/functional/test07_json_merge.py index de70052f..dec51a95 100644 --- a/tests/functional/test07_json_merge.py +++ b/tests/functional/test07_json_merge.py @@ -11,7 +11,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument # type: ignore diff --git a/tests/functional/test08_json_finalization.py b/tests/functional/test08_json_finalization.py index a05daccc..a6ff570e 100644 --- a/tests/functional/test08_json_finalization.py +++ b/tests/functional/test08_json_finalization.py @@ -32,7 +32,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument, extractJsonString, repairBrokenJson # type: ignore diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 3c85460e..9a42c04f 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User -from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.workflows.automation import chatStart +import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 19bdb12f..1ef6b678 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User -from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.workflows.automation import chatStart +import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester10: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 64c8f93e..f443ff61 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,10 +23,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User -from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.workflows.automation import chatStart +import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class CodeGenerationFormatsTester11: diff --git a/tests/functional/test12_json_split_merge.py b/tests/functional/test12_json_split_merge.py index 4dac56cb..2632ed2e 100644 --- a/tests/functional/test12_json_split_merge.py +++ b/tests/functional/test12_json_split_merge.py @@ -20,7 +20,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import JSON merger from workflow tools -from modules.services.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger +from modules.features.aichat.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger from modules.shared.jsonContinuation import getContexts diff --git a/tests/functional/test_kpi_full.py b/tests/functional/test_kpi_full.py index 32e40f07..4457b3e3 100644 --- a/tests/functional/test_kpi_full.py +++ b/tests/functional/test_kpi_full.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState # Load actual JSON response diff --git a/tests/functional/test_kpi_incomplete.py b/tests/functional/test_kpi_incomplete.py index f6e60aa0..90cdcdcb 100644 --- a/tests/functional/test_kpi_incomplete.py +++ b/tests/functional/test_kpi_incomplete.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState from modules.shared.jsonUtils import extractJsonString, repairBrokenJson diff --git a/tests/functional/test_kpi_path.py b/tests/functional/test_kpi_path.py index 0c54e3c2..66e7293b 100644 --- a/tests/functional/test_kpi_path.py +++ b/tests/functional/test_kpi_path.py @@ -10,7 +10,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # Test JSON matching the actual response test_json = { diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index a2b69576..26552008 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/options/test_frontend_options_types.py b/tests/unit/options/test_frontend_options_types.py deleted file mode 100644 index fbb7ed75..00000000 --- a/tests/unit/options/test_frontend_options_types.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Unit tests for frontend_options type system and utilities. -Tests type validation, format detection, and utility functions. -""" - -import pytest -from modules.shared.frontendOptionsTypes import ( - FrontendOptions, - OptionItem, - isStringReference, - isStaticList, - validateFrontendOptions, - getOptionsName, - getStaticOptions -) - - -class TestFrontendOptionsTypes: - """Test frontend_options type system.""" - - def testIsStringReference(self): - """Test string reference detection.""" - assert isStringReference("user.role") is True - assert isStringReference("auth.authority") is True - assert isStringReference("") is True # Empty string is still a string - - assert isStringReference([]) is False - assert isStringReference([{"value": "a"}]) is False - assert isStringReference(None) is False - - def testIsStaticList(self): - """Test static list detection.""" - assert isStaticList([]) is True - assert isStaticList([{"value": "a", "label": {"en": "A"}}]) is True - - assert isStaticList("user.role") is False - assert isStaticList(None) is False - - def testValidateFrontendOptionsString(self): - """Test validation of string references.""" - assert validateFrontendOptions("user.role") is True - assert validateFrontendOptions("auth.authority") is True - assert validateFrontendOptions("") is False # Empty string is invalid - assert validateFrontendOptions(" ") is False # Whitespace-only is invalid - - def testValidateFrontendOptionsStaticList(self): - """Test validation of static lists.""" - # Valid static list - validList = [ - {"value": "a", "label": {"en": "All", "fr": "Tous"}}, - {"value": "m", "label": {"en": "My", "fr": "Mes"}} - ] - assert validateFrontendOptions(validList) is True - - # Empty list is valid - assert validateFrontendOptions([]) is True - - # Missing value key - invalidList1 = [{"label": {"en": "Test"}}] - assert validateFrontendOptions(invalidList1) is False - - # Missing label key - invalidList2 = [{"value": "a"}] - assert validateFrontendOptions(invalidList2) is False - - # Label is not a dict - invalidList3 = [{"value": "a", "label": "not a dict"}] - assert validateFrontendOptions(invalidList3) is False - - # Not a list or string - assert validateFrontendOptions(None) is False - assert validateFrontendOptions(123) is False - assert validateFrontendOptions({}) is False - - def testGetOptionsName(self): - """Test getting options name from string reference.""" - assert getOptionsName("user.role") == "user.role" - assert getOptionsName("auth.authority") == "auth.authority" - - # Should raise ValueError for non-string - with pytest.raises(ValueError): - getOptionsName([]) - - with pytest.raises(ValueError): - getOptionsName(None) - - def testGetStaticOptions(self): - """Test getting static options list.""" - options = [ - {"value": "a", "label": {"en": "All"}}, - {"value": "m", "label": {"en": "My"}} - ] - assert getStaticOptions(options) == options - - # Should raise ValueError for non-list - with pytest.raises(ValueError): - getStaticOptions("user.role") - - with pytest.raises(ValueError): - getStaticOptions(None) - - def testTypeAliases(self): - """Test that type aliases are properly defined.""" - # FrontendOptions should accept both str and List[OptionItem] - stringRef: FrontendOptions = "user.role" - staticList: FrontendOptions = [{"value": "a", "label": {"en": "A"}}] - - assert isinstance(stringRef, str) - assert isinstance(staticList, list) - - # OptionItem should be Dict[str, Any] - optionItem: OptionItem = {"value": "test", "label": {"en": "Test"}} - assert isinstance(optionItem, dict) - assert "value" in optionItem - assert "label" in optionItem diff --git a/tests/unit/options/test_main_options.py b/tests/unit/options/test_main_options.py deleted file mode 100644 index 1182fb9c..00000000 --- a/tests/unit/options/test_main_options.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Unit tests for Dynamic Options API (mainDynamicOptions.py). -Tests option retrieval, validation, and context-aware options. -""" - -import pytest -from unittest.mock import Mock, patch -from modules.features.dynamicOptions.mainDynamicOptions import ( - getOptions, - getAvailableOptionsNames, - STANDARD_ROLES, - AUTH_AUTHORITY_OPTIONS, - CONNECTION_STATUS_OPTIONS -) -from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority - - -class TestMainOptions: - """Test Options API functionality.""" - - def testGetOptionsUserRole(self): - """Test getting user role options.""" - options = getOptions("user.role") - - assert isinstance(options, list) - assert len(options) == 4 # sysadmin, admin, user, viewer - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - assert isinstance(option["label"], dict) - assert "en" in option["label"] - assert "fr" in option["label"] - - # Check specific values - values = [opt["value"] for opt in options] - assert "sysadmin" in values - assert "admin" in values - assert "user" in values - assert "viewer" in values - - def testGetOptionsAuthAuthority(self): - """Test getting auth authority options.""" - options = getOptions("auth.authority") - - assert isinstance(options, list) - assert len(options) == 3 # local, google, msft - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - - # Check specific values - values = [opt["value"] for opt in options] - assert "local" in values - assert "google" in values - assert "msft" in values - - def testGetOptionsConnectionStatus(self): - """Test getting connection status options.""" - options = getOptions("connection.status") - - assert isinstance(options, list) - assert len(options) == 5 # active, expired, revoked, pending, error - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - - # Check specific values - values = [opt["value"] for opt in options] - assert "active" in values - assert "expired" in values - assert "revoked" in values - assert "pending" in values - assert "error" in values - - def testGetOptionsUserConnection(self): - """Test getting user connection options (context-aware).""" - # Without currentUser, should return empty list - options = getOptions("user.connection") - assert options == [] - - # With currentUser but no connections - user = User( - id="user1", - username="testuser", - roleLabels=["user"], - mandateId="mandate1" - ) - - with patch('modules.features.dynamicOptions.mainDynamicOptions.getInterface') as mockGetInterface: - mockInterface = Mock() - mockInterface.getUserConnections.return_value = [] - mockGetInterface.return_value = mockInterface - - options = getOptions("user.connection", currentUser=user) - assert options == [] - - def testGetOptionsUserConnectionWithData(self): - """Test getting user connection options with actual connections.""" - user = User( - id="user1", - username="testuser", - roleLabels=["user"], - mandateId="mandate1" - ) - - # Mock connections - mockConn1 = Mock(spec=UserConnection) - mockConn1.id = "conn1" - mockConn1.authority = AuthAuthority.GOOGLE - mockConn1.externalUsername = "user@example.com" - mockConn1.externalId = None - - mockConn2 = Mock(spec=UserConnection) - mockConn2.id = "conn2" - mockConn2.authority = AuthAuthority.MSFT - mockConn2.externalUsername = None - mockConn2.externalId = "external-id-123" - - with patch('modules.features.dynamicOptions.mainDynamicOptions.getInterface') as mockGetInterface: - mockInterface = Mock() - mockInterface.getUserConnections.return_value = [mockConn1, mockConn2] - mockGetInterface.return_value = mockInterface - - options = getOptions("user.connection", currentUser=user) - - assert len(options) == 2 - assert options[0]["value"] == "conn1" - assert options[1]["value"] == "conn2" - - # Check labels contain authority and username/id - assert "google" in options[0]["label"]["en"].lower() - assert "user@example.com" in options[0]["label"]["en"] - - def testGetOptionsCaseInsensitive(self): - """Test that options name matching is case-insensitive.""" - options1 = getOptions("user.role") - options2 = getOptions("USER.ROLE") - options3 = getOptions("User.Role") - - assert options1 == options2 == options3 - - def testGetOptionsUnknown(self): - """Test that unknown options name raises ValueError.""" - with pytest.raises(ValueError, match="Unknown options name"): - getOptions("unknown.options") - - def testGetAvailableOptionsNames(self): - """Test getting list of available options names.""" - names = getAvailableOptionsNames() - - assert isinstance(names, list) - assert "user.role" in names - assert "auth.authority" in names - assert "connection.status" in names - assert "user.connection" in names - assert len(names) == 4 - - def testStandardRolesConstant(self): - """Test that STANDARD_ROLES constant is properly defined.""" - assert isinstance(STANDARD_ROLES, list) - assert len(STANDARD_ROLES) == 4 - - for role in STANDARD_ROLES: - assert "value" in role - assert "label" in role - - def testAuthAuthorityOptionsConstant(self): - """Test that AUTH_AUTHORITY_OPTIONS constant is properly defined.""" - assert isinstance(AUTH_AUTHORITY_OPTIONS, list) - assert len(AUTH_AUTHORITY_OPTIONS) == 3 - - def testConnectionStatusOptionsConstant(self): - """Test that CONNECTION_STATUS_OPTIONS constant is properly defined.""" - assert isinstance(CONNECTION_STATUS_OPTIONS, list) - assert len(CONNECTION_STATUS_OPTIONS) == 5 # active, expired, revoked, pending, error diff --git a/tests/unit/services/test_json_extraction_merging.py b/tests/unit/services/test_json_extraction_merging.py index 07ecfa4b..644b7140 100644 --- a/tests/unit/services/test_json_extraction_merging.py +++ b/tests/unit/services/test_json_extraction_merging.py @@ -14,7 +14,7 @@ import os sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..')) from modules.datamodels.datamodelExtraction import ContentPart -from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService def test_detects_json_with_code_fences(): diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index ae502397..d649826e 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 09f6e92c..dfc46be1 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel From f02ebead7c92446f412208d54955777fa0f1c851 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 22 Jan 2026 18:52:04 +0100 Subject: [PATCH 13/32] dyn options in api --- modules/datamodels/datamodelUam.py | 6 +- .../trustee/datamodelFeatureTrustee.py | 26 +- .../features/trustee/routeFeatureTrustee.py | 83 ++++++ modules/routes/routeDataConnections.py | 46 ++++ modules/routes/routeDataUsers.py | 42 +++ modules/shared/frontendOptionsTypes.py | 138 ---------- tests/integration/options/test_options_api.py | 243 ------------------ 7 files changed, 187 insertions(+), 397 deletions(-) delete mode 100644 modules/shared/frontendOptionsTypes.py delete mode 100644 tests/integration/options/test_options_api.py diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index 96a99fee..42659159 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -100,11 +100,11 @@ registerModelLabels( class UserConnection(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the connection", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user this connection belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - authority: AuthAuthority = Field(description="Authentication authority", json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "auth.authority"}) + authority: AuthAuthority = Field(description="Authentication authority", json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "/api/connections/authorities/options"}) externalId: str = Field(description="User ID in the external system", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) externalUsername: str = Field(description="Username in the external system", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) externalEmail: Optional[EmailStr] = Field(None, description="Email in the external system", json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": False}) - status: ConnectionStatus = Field(default=ConnectionStatus.ACTIVE, description="Connection status", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": "connection.status"}) + status: ConnectionStatus = Field(default=ConnectionStatus.ACTIVE, description="Connection status", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": "/api/connections/statuses/options"}) connectedAt: float = Field(default_factory=getUtcTimestamp, description="When the connection was established (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) lastChecked: float = Field(default_factory=getUtcTimestamp, description="When the connection was last verified (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) expiresAt: Optional[float] = Field(None, description="When the connection expires (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) @@ -198,7 +198,7 @@ class User(BaseModel): authenticationAuthority: AuthAuthority = Field( default=AuthAuthority.LOCAL, description="Primary authentication authority", - json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "auth.authority"} + json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "/api/connections/authorities/options"} ) diff --git a/modules/features/trustee/datamodelFeatureTrustee.py b/modules/features/trustee/datamodelFeatureTrustee.py index e2d9b261..c64ec506 100644 --- a/modules/features/trustee/datamodelFeatureTrustee.py +++ b/modules/features/trustee/datamodelFeatureTrustee.py @@ -138,7 +138,7 @@ class TrusteeAccess(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeOrganisation" + "frontend_options": "/api/trustee/{instanceId}/organisations/options" } ) roleId: str = Field( @@ -147,7 +147,7 @@ class TrusteeAccess(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeRole" + "frontend_options": "/api/trustee/{instanceId}/roles/options" } ) userId: str = Field( @@ -156,7 +156,7 @@ class TrusteeAccess(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "User" + "frontend_options": "/api/users/options" } ) contractId: Optional[str] = Field( @@ -166,7 +166,7 @@ class TrusteeAccess(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": False, - "frontend_options": "TrusteeContract", + "frontend_options": "/api/trustee/{instanceId}/contracts/options", "frontend_depends_on": "organisationId" } ) @@ -223,7 +223,7 @@ class TrusteeContract(BaseModel): "frontend_type": "select", "frontend_readonly": False, # Editable at creation, then readonly "frontend_required": True, - "frontend_options": "TrusteeOrganisation" + "frontend_options": "/api/trustee/{instanceId}/organisations/options" } ) label: str = Field( @@ -295,7 +295,7 @@ class TrusteeDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeOrganisation" + "frontend_options": "/api/trustee/{instanceId}/organisations/options" } ) contractId: str = Field( @@ -304,7 +304,7 @@ class TrusteeDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeContract", + "frontend_options": "/api/trustee/{instanceId}/contracts/options", "frontend_depends_on": "organisationId" } ) @@ -394,7 +394,7 @@ class TrusteePosition(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeOrganisation" + "frontend_options": "/api/trustee/{instanceId}/organisations/options" } ) contractId: str = Field( @@ -403,7 +403,7 @@ class TrusteePosition(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeContract", + "frontend_options": "/api/trustee/{instanceId}/contracts/options", "frontend_depends_on": "organisationId" } ) @@ -580,7 +580,7 @@ class TrusteePositionDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeOrganisation" + "frontend_options": "/api/trustee/{instanceId}/organisations/options" } ) contractId: str = Field( @@ -589,7 +589,7 @@ class TrusteePositionDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeContract", + "frontend_options": "/api/trustee/{instanceId}/contracts/options", "frontend_depends_on": "organisationId" } ) @@ -599,7 +599,7 @@ class TrusteePositionDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteeDocument", + "frontend_options": "/api/trustee/{instanceId}/documents/options", "frontend_depends_on": "contractId" } ) @@ -609,7 +609,7 @@ class TrusteePositionDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "TrusteePosition", + "frontend_options": "/api/trustee/{instanceId}/positions/options", "frontend_depends_on": "contractId" } ) diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index bee52513..196c8b18 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -115,6 +115,89 @@ async def _validateInstanceAccess(instanceId: str, context: RequestContext) -> s return str(instance.mandateId) +# ============================================================================ +# OPTIONS ENDPOINTS (for dropdowns) +# ============================================================================ + +@router.get("/{instanceId}/organisations/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getOrganisationOptions( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get organisation options for select dropdowns. Returns: [{ value, label }]""" + mandateId = await _validateInstanceAccess(instanceId, context) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.getAllOrganisations(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": org.id, "label": org.label or org.id} for org in items] + + +@router.get("/{instanceId}/roles/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getRoleOptions( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get role options for select dropdowns. Returns: [{ value, label }]""" + mandateId = await _validateInstanceAccess(instanceId, context) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.getAllRoles(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": role.id, "label": role.desc or role.id} for role in items] + + +@router.get("/{instanceId}/contracts/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getContractOptions( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get contract options for select dropdowns. Returns: [{ value, label }]""" + mandateId = await _validateInstanceAccess(instanceId, context) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.getAllContracts(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": c.id, "label": c.label or c.name or c.id} for c in items] + + +@router.get("/{instanceId}/documents/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getDocumentOptions( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get document options for select dropdowns. Returns: [{ value, label }]""" + mandateId = await _validateInstanceAccess(instanceId, context) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.getAllDocuments(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": d.id, "label": d.name or d.id} for d in items] + + +@router.get("/{instanceId}/positions/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getPositionOptions( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get position options for select dropdowns. Returns: [{ value, label }]""" + mandateId = await _validateInstanceAccess(instanceId, context) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.getAllPositions(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": p.id, "label": p.title or p.id} for p in items] + + +# ============================================================================ +# CRUD ENDPOINTS +# ============================================================================ + # ===== Organisation Routes ===== @router.get("/{instanceId}/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index f39b6638..c7af510e 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -92,6 +92,52 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) + +# ============================================================================ +# OPTIONS ENDPOINTS (for dropdowns) +# ============================================================================ + +@router.get("/statuses/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getConnectionStatusOptions( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> List[Dict[str, Any]]: + """ + Get connection status options for select dropdowns. + Returns standardized format: [{ value, label }] + """ + return [ + {"value": status.value, "label": status.value.capitalize()} + for status in ConnectionStatus + ] + + +@router.get("/authorities/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getAuthAuthorityOptions( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> List[Dict[str, Any]]: + """ + Get authentication authority options for select dropdowns. + Returns standardized format: [{ value, label }] + """ + authorityLabels = { + "local": "Local", + "google": "Google", + "msft": "Microsoft" + } + return [ + {"value": auth.value, "label": authorityLabels.get(auth.value, auth.value)} + for auth in AuthAuthority + ] + + +# ============================================================================ +# CRUD ENDPOINTS +# ============================================================================ + @router.get("/", response_model=PaginatedResponse[UserConnection]) @limiter.limit("30/minute") async def get_connections( diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index dab8bde9..809b3521 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -145,6 +145,48 @@ router = APIRouter( responses={404: {"description": "Not found"}} ) + +# ============================================================================ +# OPTIONS ENDPOINTS (for dropdowns) +# ============================================================================ + +@router.get("/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getUserOptions( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + Get user options for select dropdowns. + MULTI-TENANT: mandateId from X-Mandate-Id header determines scope. + Returns standardized format: [{ value, label }] + """ + try: + appInterface = interfaceDbApp.getInterface(context.user) + + if context.mandateId: + result = appInterface.getUsersByMandate(str(context.mandateId), None) + users = result.items if hasattr(result, 'items') else result + elif context.isSysAdmin: + users = appInterface.getAllUsers() + else: + raise HTTPException(status_code=403, detail="Access denied") + + return [ + {"value": user.id, "label": user.fullName or user.username or user.email or user.id} + for user in users + ] + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting user options: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to get user options: {str(e)}") + + +# ============================================================================ +# CRUD ENDPOINTS +# ============================================================================ + @router.get("/", response_model=PaginatedResponse[User]) @limiter.limit("30/minute") async def get_users( diff --git a/modules/shared/frontendOptionsTypes.py b/modules/shared/frontendOptionsTypes.py deleted file mode 100644 index 4863a04c..00000000 --- a/modules/shared/frontendOptionsTypes.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Type definitions and utilities for frontend_options attribute. - -The frontend_options attribute supports two formats: -1. Static List: A list of option dictionaries for static options -2. String Reference: A string identifier that references dynamic options from /api/options/{optionsName} -""" - -from typing import List, Dict, Any, Union - -try: - from typing import TypeAlias # Python 3.10+ -except ImportError: - from typing_extensions import TypeAlias # Python < 3.10 - -# Type definition for a single option item -OptionItem: TypeAlias = Dict[str, Any] -""" -Single option item format: -{ - "value": str, # The value to be stored/returned - "label": { # Multilingual labels - "en": str, - "fr": str, - ... - } -} -""" - -# Type definition for frontend_options - can be either a list or string reference -FrontendOptions: TypeAlias = Union[List[OptionItem], str] -""" -frontend_options can be either: -1. List[OptionItem]: Static list of options - Example: [{"value": "a", "label": {"en": "All", "fr": "Tous"}}] - -2. str: String reference to dynamic options API - Example: "user.role" -> Frontend fetches from /api/options/user.role -""" - - -def isStringReference(frontendOptions: FrontendOptions) -> bool: - """ - Check if frontend_options is a string reference (dynamic) or a list (static). - - Args: - frontendOptions: The frontend_options value to check - - Returns: - True if it's a string reference, False if it's a list - """ - return isinstance(frontendOptions, str) - - -def isStaticList(frontendOptions: FrontendOptions) -> bool: - """ - Check if frontend_options is a static list or a string reference. - - Args: - frontendOptions: The frontend_options value to check - - Returns: - True if it's a static list, False if it's a string reference - """ - return isinstance(frontendOptions, list) - - -def validateFrontendOptions(frontendOptions: FrontendOptions) -> bool: - """ - Validate that frontend_options is in the correct format. - - Args: - frontendOptions: The frontend_options value to validate - - Returns: - True if valid, False otherwise - """ - if isinstance(frontendOptions, str): - # String reference: should be a non-empty string - return bool(frontendOptions.strip()) - - elif isinstance(frontendOptions, list): - # Static list: should contain option dictionaries - if not frontendOptions: - return True # Empty list is valid (no options) - - for option in frontendOptions: - if not isinstance(option, dict): - return False - if "value" not in option: - return False - if "label" not in option: - return False - if not isinstance(option["label"], dict): - return False - - return True - - else: - return False - - -def getOptionsName(frontendOptions: FrontendOptions) -> str: - """ - Get the options name from a string reference. - - Args: - frontendOptions: The frontend_options value (must be a string reference) - - Returns: - The options name (e.g., "user.role") - - Raises: - ValueError: If frontendOptions is not a string reference - """ - if not isStringReference(frontendOptions): - raise ValueError(f"frontend_options is not a string reference: {type(frontendOptions)}") - return frontendOptions - - -def getStaticOptions(frontendOptions: FrontendOptions) -> List[OptionItem]: - """ - Get the static options list. - - Args: - frontendOptions: The frontend_options value (must be a static list) - - Returns: - The list of option items - - Raises: - ValueError: If frontendOptions is not a static list - """ - if not isStaticList(frontendOptions): - raise ValueError(f"frontend_options is not a static list: {type(frontendOptions)}") - return frontendOptions diff --git a/tests/integration/options/test_options_api.py b/tests/integration/options/test_options_api.py deleted file mode 100644 index 7007bf2a..00000000 --- a/tests/integration/options/test_options_api.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Integration tests for Options API endpoints. -Tests the actual API endpoints with real database connections. -""" - -import pytest -import secrets -from fastapi.testclient import TestClient -from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbApp import getRootInterface - - -@pytest.fixture -def app(): - """Create FastAPI app instance for testing.""" - from app import app as fastapi_app - return fastapi_app - - -@pytest.fixture -def testClient(app): - """Create test client for API testing.""" - return TestClient(app) - - -@pytest.fixture -def csrfToken(): - """Generate a valid CSRF token for testing.""" - # Generate a hex string between 16-64 characters (CSRF validation requirement) - return secrets.token_hex(16) # 32 character hex string - - -@pytest.fixture -def testUser() -> User: - """Create a test user for API testing.""" - # Use getRootInterface for system operations like user creation - # The root interface automatically uses the root mandate - rootInterface = getRootInterface() - user = rootInterface.createUser( - username="testuser_options", - email="testuser_options@example.com", - password="testpass123", - roleLabels=["user"] - ) - return user - - -class TestOptionsAPI: - """Test Options API endpoints.""" - - def testGetOptionsUserRole(self, testClient, testUser, csrfToken): - """Test GET /api/options/user.role endpoint.""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get options - response = testClient.get( - "/api/options/user.role", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 200 - options = response.json() - - assert isinstance(options, list) - assert len(options) >= 4 # At least sysadmin, admin, user, viewer - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - assert isinstance(option["label"], dict) - - # Check specific values - values = [opt["value"] for opt in options] - assert "sysadmin" in values - assert "admin" in values - assert "user" in values - assert "viewer" in values - - def testGetOptionsAuthAuthority(self, testClient, testUser, csrfToken): - """Test GET /api/options/auth.authority endpoint.""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get options - response = testClient.get( - "/api/options/auth.authority", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 200 - options = response.json() - - assert isinstance(options, list) - assert len(options) == 3 # local, google, msft - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - - # Check specific values - values = [opt["value"] for opt in options] - assert "local" in values - assert "google" in values - assert "msft" in values - - def testGetOptionsConnectionStatus(self, testClient, testUser, csrfToken): - """Test GET /api/options/connection.status endpoint.""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get options - response = testClient.get( - "/api/options/connection.status", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 200 - options = response.json() - - assert isinstance(options, list) - assert len(options) >= 4 # active, inactive, expired, pending, revoked, error - - # Check structure - for option in options: - assert "value" in option - assert "label" in option - - def testGetOptionsUserConnection(self, testClient, testUser, csrfToken): - """Test GET /api/options/user.connection endpoint (context-aware).""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get options (should return empty list if no connections) - response = testClient.get( - "/api/options/user.connection", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 200 - options = response.json() - - # Should return a list (may be empty) - assert isinstance(options, list) - - def testGetOptionsList(self, testClient, testUser, csrfToken): - """Test GET /api/options/ endpoint (list all available options).""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get available options names - response = testClient.get( - "/api/options/", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 200 - optionsNames = response.json() - - assert isinstance(optionsNames, list) - assert "user.role" in optionsNames - assert "auth.authority" in optionsNames - assert "connection.status" in optionsNames - assert "user.connection" in optionsNames - - def testGetOptionsUnknown(self, testClient, testUser, csrfToken): - """Test GET /api/options/unknown.options endpoint (should return 400).""" - # Get auth token (stored in cookie) - response = testClient.post( - "/api/local/login", - data={"username": testUser.username, "password": "testpass123"}, - headers={"X-CSRF-Token": csrfToken} - ) - assert response.status_code == 200 - - # Extract token from cookie for Bearer header - token = response.cookies.get("auth_token") - assert token is not None - - # Get unknown options (should return error) - response = testClient.get( - "/api/options/unknown.options", - headers={"Authorization": f"Bearer {token}"} - ) - - assert response.status_code == 400 - - def testGetOptionsUnauthorized(self, testClient): - """Test GET /api/options/user.role without authentication.""" - # Try to get options without auth token - response = testClient.get("/api/options/user.role") - - # Should require authentication - assert response.status_code == 401 From bb9630d6c4f676d74e36459c23edf7ebb444cd32 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 22 Jan 2026 21:11:25 +0100 Subject: [PATCH 14/32] fixed imports --- .../aichat/aicore/aicoreBase.py | 0 .../aichat/aicore/aicoreModelRegistry.py | 0 .../aichat/aicore/aicoreModelSelector.py | 0 .../aichat/aicore/aicorePluginAnthropic.py | 0 .../aichat/aicore/aicorePluginInternal.py | 0 .../aichat/aicore/aicorePluginOpenai.py | 0 .../aichat/aicore/aicorePluginPerplexity.py | 0 .../aichat/aicore/aicorePluginTavily.py | 0 .../aichat/datamodelFeatureAiChat.py | 0 .../aichat/interfaceFeatureAiChat.py | 0 modules/{features => }/aichat/mainAiChat.py | 0 .../aichat/routeFeatureAiChat.py | 0 .../aichat/serviceAi/mainServiceAi.py | 12 +- .../aichat/serviceAi/merge_1.txt | 0 .../aichat/serviceAi/subAiCallLooping-flow.md | 0 .../aichat/serviceAi/subAiCallLooping.py | 0 .../aichat/serviceAi/subContentExtraction.py | 2 +- .../aichat/serviceAi/subDocumentIntents.py | 2 +- .../aichat/serviceAi/subJsonMerger.py | 0 .../serviceAi/subJsonResponseHandling.py | 0 .../aichat/serviceAi/subLoopingUseCases.py | 0 .../aichat/serviceAi/subResponseParsing.py | 0 .../aichat/serviceAi/subStructureFilling.py | 2 +- .../serviceAi/subStructureGeneration.py | 2 +- .../aichat/serviceExtraction/__init__.py | 0 .../serviceExtraction/chunking/__init__.py | 0 .../chunking/chunkerImage.py | 0 .../chunking/chunkerStructure.py | 0 .../chunking/chunkerTable.py | 0 .../serviceExtraction/chunking/chunkerText.py | 0 .../serviceExtraction/extractors/__init__.py | 0 .../extractors/extractorBinary.py | 0 .../extractors/extractorCsv.py | 0 .../extractors/extractorDocx.py | 0 .../extractors/extractorHtml.py | 0 .../extractors/extractorImage.py | 0 .../extractors/extractorJson.py | 0 .../extractors/extractorPdf.py | 0 .../extractors/extractorPptx.py | 0 .../extractors/extractorSql.py | 0 .../extractors/extractorText.py | 0 .../extractors/extractorXlsx.py | 0 .../extractors/extractorXml.py | 0 .../mainServiceExtraction.py | 6 +- .../serviceExtraction/merging/__init__.py | 0 .../merging/mergerDefault.py | 0 .../serviceExtraction/merging/mergerTable.py | 0 .../serviceExtraction/merging/mergerText.py | 0 .../aichat/serviceExtraction/subMerger.py | 0 .../aichat/serviceExtraction/subPipeline.py | 0 .../subPromptBuilderExtraction.py | 2 +- .../aichat/serviceExtraction/subRegistry.py | 2 +- .../aichat/serviceExtraction/subUtils.py | 0 .../mainServiceGeneration.py | 12 +- .../serviceGeneration/paths/codePath.py | 2 +- .../serviceGeneration/paths/documentPath.py | 0 .../serviceGeneration/paths/imagePath.py | 0 .../renderers/codeRendererBaseTemplate.py | 0 .../renderers/documentRendererBaseTemplate.py | 0 .../serviceGeneration/renderers/registry.py | 0 .../renderers/rendererCodeCsv.py | 0 .../renderers/rendererCodeJson.py | 0 .../renderers/rendererCodeXml.py | 0 .../renderers/rendererCsv.py | 0 .../renderers/rendererDocx.py | 0 .../renderers/rendererHtml.py | 0 .../renderers/rendererImage.py | 0 .../renderers/rendererJson.py | 0 .../renderers/rendererMarkdown.py | 0 .../renderers/rendererPdf.py | 0 .../renderers/rendererPptx.py | 0 .../renderers/rendererText.py | 0 .../renderers/rendererXlsx.py | 0 .../serviceGeneration/subContentGenerator.py | 2 +- .../serviceGeneration/subContentIntegrator.py | 0 .../serviceGeneration/subDocumentUtility.py | 0 .../aichat/serviceGeneration/subJsonSchema.py | 0 .../subPromptBuilderGeneration.py | 0 .../subStructureGenerator.py | 0 .../aichat/serviceWeb/mainServiceWeb.py | 0 .../datamodels/datamodelWorkflowActions.py | 2 +- .../automation/routeFeatureAutomation.py | 4 +- modules/features/chatbot/mainChatbot.py | 4 +- .../interfaceFeatureNeutralizer.py | 176 ++ .../mainServiceNeutralization.py | 58 +- modules/interfaces/interfaceAiObjects.py | 4 +- modules/interfaces/interfaceDbApp.py | 113 -- modules/routes/routeAdminAutomationEvents.py | 4 +- modules/routes/routeDataWorkflows.py | 6 +- modules/security/passwordUtils.py | 54 + modules/services/__init__.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 2 +- modules/shared/__init__.py | 19 + modules/workflows/automation/mainWorkflow.py | 2 +- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 6 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 8 +- modules/workflows/workflowManager.py | 10 +- scripts/import_analysis.csv | 1753 ++++++++--------- scripts/script_analyze_imports.py | 30 +- tests/functional/test01_ai_model_selection.py | 6 +- tests/functional/test02_ai_models.py | 16 +- tests/functional/test03_ai_operations.py | 12 +- tests/functional/test04_ai_behavior.py | 4 +- .../test05_workflow_with_documents.py | 4 +- .../test06_workflow_prompt_variations.py | 4 +- tests/functional/test07_json_merge.py | 2 +- tests/functional/test08_json_finalization.py | 2 +- .../test09_document_generation_formats.py | 4 +- .../test10_document_generation_formats.py | 4 +- .../test11_code_generation_formats.py | 4 +- tests/functional/test12_json_split_merge.py | 2 +- tests/functional/test_kpi_full.py | 2 +- tests/functional/test_kpi_incomplete.py | 2 +- tests/functional/test_kpi_path.py | 2 +- .../workflows/test_workflow_execution.py | 2 +- .../services/test_json_extraction_merging.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- 161 files changed, 1318 insertions(+), 1165 deletions(-) rename modules/{features => }/aichat/aicore/aicoreBase.py (100%) rename modules/{features => }/aichat/aicore/aicoreModelRegistry.py (100%) rename modules/{features => }/aichat/aicore/aicoreModelSelector.py (100%) rename modules/{features => }/aichat/aicore/aicorePluginAnthropic.py (100%) rename modules/{features => }/aichat/aicore/aicorePluginInternal.py (100%) rename modules/{features => }/aichat/aicore/aicorePluginOpenai.py (100%) rename modules/{features => }/aichat/aicore/aicorePluginPerplexity.py (100%) rename modules/{features => }/aichat/aicore/aicorePluginTavily.py (100%) rename modules/{features => }/aichat/datamodelFeatureAiChat.py (100%) rename modules/{features => }/aichat/interfaceFeatureAiChat.py (100%) rename modules/{features => }/aichat/mainAiChat.py (100%) rename modules/{features => }/aichat/routeFeatureAiChat.py (100%) rename modules/{features => }/aichat/serviceAi/mainServiceAi.py (98%) rename modules/{features => }/aichat/serviceAi/merge_1.txt (100%) rename modules/{features => }/aichat/serviceAi/subAiCallLooping-flow.md (100%) rename modules/{features => }/aichat/serviceAi/subAiCallLooping.py (100%) rename modules/{features => }/aichat/serviceAi/subContentExtraction.py (99%) rename modules/{features => }/aichat/serviceAi/subDocumentIntents.py (99%) rename modules/{features => }/aichat/serviceAi/subJsonMerger.py (100%) rename modules/{features => }/aichat/serviceAi/subJsonResponseHandling.py (100%) rename modules/{features => }/aichat/serviceAi/subLoopingUseCases.py (100%) rename modules/{features => }/aichat/serviceAi/subResponseParsing.py (100%) rename modules/{features => }/aichat/serviceAi/subStructureFilling.py (99%) rename modules/{features => }/aichat/serviceAi/subStructureGeneration.py (99%) rename modules/{features => }/aichat/serviceExtraction/__init__.py (100%) rename modules/{features => }/aichat/serviceExtraction/chunking/__init__.py (100%) rename modules/{features => }/aichat/serviceExtraction/chunking/chunkerImage.py (100%) rename modules/{features => }/aichat/serviceExtraction/chunking/chunkerStructure.py (100%) rename modules/{features => }/aichat/serviceExtraction/chunking/chunkerTable.py (100%) rename modules/{features => }/aichat/serviceExtraction/chunking/chunkerText.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/__init__.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorBinary.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorCsv.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorDocx.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorHtml.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorImage.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorJson.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorPdf.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorPptx.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorSql.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorText.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorXlsx.py (100%) rename modules/{features => }/aichat/serviceExtraction/extractors/extractorXml.py (100%) rename modules/{features => }/aichat/serviceExtraction/mainServiceExtraction.py (99%) rename modules/{features => }/aichat/serviceExtraction/merging/__init__.py (100%) rename modules/{features => }/aichat/serviceExtraction/merging/mergerDefault.py (100%) rename modules/{features => }/aichat/serviceExtraction/merging/mergerTable.py (100%) rename modules/{features => }/aichat/serviceExtraction/merging/mergerText.py (100%) rename modules/{features => }/aichat/serviceExtraction/subMerger.py (100%) rename modules/{features => }/aichat/serviceExtraction/subPipeline.py (100%) rename modules/{features => }/aichat/serviceExtraction/subPromptBuilderExtraction.py (98%) rename modules/{features => }/aichat/serviceExtraction/subRegistry.py (99%) rename modules/{features => }/aichat/serviceExtraction/subUtils.py (100%) rename modules/{features => }/aichat/serviceGeneration/mainServiceGeneration.py (98%) rename modules/{features => }/aichat/serviceGeneration/paths/codePath.py (99%) rename modules/{features => }/aichat/serviceGeneration/paths/documentPath.py (100%) rename modules/{features => }/aichat/serviceGeneration/paths/imagePath.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/registry.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererCodeCsv.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererCodeJson.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererCodeXml.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererCsv.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererDocx.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererHtml.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererImage.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererJson.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererMarkdown.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererPdf.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererPptx.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererText.py (100%) rename modules/{features => }/aichat/serviceGeneration/renderers/rendererXlsx.py (100%) rename modules/{features => }/aichat/serviceGeneration/subContentGenerator.py (99%) rename modules/{features => }/aichat/serviceGeneration/subContentIntegrator.py (100%) rename modules/{features => }/aichat/serviceGeneration/subDocumentUtility.py (100%) rename modules/{features => }/aichat/serviceGeneration/subJsonSchema.py (100%) rename modules/{features => }/aichat/serviceGeneration/subPromptBuilderGeneration.py (100%) rename modules/{features => }/aichat/serviceGeneration/subStructureGenerator.py (100%) rename modules/{features => }/aichat/serviceWeb/mainServiceWeb.py (100%) create mode 100644 modules/features/neutralizer/interfaceFeatureNeutralizer.py create mode 100644 modules/security/passwordUtils.py create mode 100644 modules/shared/__init__.py diff --git a/modules/features/aichat/aicore/aicoreBase.py b/modules/aichat/aicore/aicoreBase.py similarity index 100% rename from modules/features/aichat/aicore/aicoreBase.py rename to modules/aichat/aicore/aicoreBase.py diff --git a/modules/features/aichat/aicore/aicoreModelRegistry.py b/modules/aichat/aicore/aicoreModelRegistry.py similarity index 100% rename from modules/features/aichat/aicore/aicoreModelRegistry.py rename to modules/aichat/aicore/aicoreModelRegistry.py diff --git a/modules/features/aichat/aicore/aicoreModelSelector.py b/modules/aichat/aicore/aicoreModelSelector.py similarity index 100% rename from modules/features/aichat/aicore/aicoreModelSelector.py rename to modules/aichat/aicore/aicoreModelSelector.py diff --git a/modules/features/aichat/aicore/aicorePluginAnthropic.py b/modules/aichat/aicore/aicorePluginAnthropic.py similarity index 100% rename from modules/features/aichat/aicore/aicorePluginAnthropic.py rename to modules/aichat/aicore/aicorePluginAnthropic.py diff --git a/modules/features/aichat/aicore/aicorePluginInternal.py b/modules/aichat/aicore/aicorePluginInternal.py similarity index 100% rename from modules/features/aichat/aicore/aicorePluginInternal.py rename to modules/aichat/aicore/aicorePluginInternal.py diff --git a/modules/features/aichat/aicore/aicorePluginOpenai.py b/modules/aichat/aicore/aicorePluginOpenai.py similarity index 100% rename from modules/features/aichat/aicore/aicorePluginOpenai.py rename to modules/aichat/aicore/aicorePluginOpenai.py diff --git a/modules/features/aichat/aicore/aicorePluginPerplexity.py b/modules/aichat/aicore/aicorePluginPerplexity.py similarity index 100% rename from modules/features/aichat/aicore/aicorePluginPerplexity.py rename to modules/aichat/aicore/aicorePluginPerplexity.py diff --git a/modules/features/aichat/aicore/aicorePluginTavily.py b/modules/aichat/aicore/aicorePluginTavily.py similarity index 100% rename from modules/features/aichat/aicore/aicorePluginTavily.py rename to modules/aichat/aicore/aicorePluginTavily.py diff --git a/modules/features/aichat/datamodelFeatureAiChat.py b/modules/aichat/datamodelFeatureAiChat.py similarity index 100% rename from modules/features/aichat/datamodelFeatureAiChat.py rename to modules/aichat/datamodelFeatureAiChat.py diff --git a/modules/features/aichat/interfaceFeatureAiChat.py b/modules/aichat/interfaceFeatureAiChat.py similarity index 100% rename from modules/features/aichat/interfaceFeatureAiChat.py rename to modules/aichat/interfaceFeatureAiChat.py diff --git a/modules/features/aichat/mainAiChat.py b/modules/aichat/mainAiChat.py similarity index 100% rename from modules/features/aichat/mainAiChat.py rename to modules/aichat/mainAiChat.py diff --git a/modules/features/aichat/routeFeatureAiChat.py b/modules/aichat/routeFeatureAiChat.py similarity index 100% rename from modules/features/aichat/routeFeatureAiChat.py rename to modules/aichat/routeFeatureAiChat.py diff --git a/modules/features/aichat/serviceAi/mainServiceAi.py b/modules/aichat/serviceAi/mainServiceAi.py similarity index 98% rename from modules/features/aichat/serviceAi/mainServiceAi.py rename to modules/aichat/serviceAi/mainServiceAi.py index e5d2605f..40e2e974 100644 --- a/modules/features/aichat/serviceAi/mainServiceAi.py +++ b/modules/aichat/serviceAi/mainServiceAi.py @@ -6,8 +6,8 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.features.aichat.datamodelFeatureAiChat import PromptPlaceholder, ChatDocument -from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.aichat.datamodelFeatureAiChat import PromptPlaceholder, ChatDocument +from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData @@ -329,7 +329,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle IMAGE_GENERATE operation type using image generation path.""" - from modules.features.aichat.serviceGeneration.paths.imagePath import ImageGenerationPath + from modules.aichat.serviceGeneration.paths.imagePath import ImageGenerationPath imagePath = ImageGenerationPath(self.services) @@ -514,7 +514,7 @@ Respond with ONLY a JSON object in this exact format: ) try: - from modules.features.aichat.serviceGeneration.mainServiceGeneration import GenerationService + from modules.aichat.serviceGeneration.mainServiceGeneration import GenerationService generationService = GenerationService(self.services) @@ -829,7 +829,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle code generation using code generation path.""" - from modules.features.aichat.serviceGeneration.paths.codePath import CodeGenerationPath + from modules.aichat.serviceGeneration.paths.codePath import CodeGenerationPath codePath = CodeGenerationPath(self.services) return await codePath.generateCode( @@ -852,7 +852,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle document generation using document generation path.""" - from modules.features.aichat.serviceGeneration.paths.documentPath import DocumentGenerationPath + from modules.aichat.serviceGeneration.paths.documentPath import DocumentGenerationPath # Set compression options for document generation options.compressPrompt = False diff --git a/modules/features/aichat/serviceAi/merge_1.txt b/modules/aichat/serviceAi/merge_1.txt similarity index 100% rename from modules/features/aichat/serviceAi/merge_1.txt rename to modules/aichat/serviceAi/merge_1.txt diff --git a/modules/features/aichat/serviceAi/subAiCallLooping-flow.md b/modules/aichat/serviceAi/subAiCallLooping-flow.md similarity index 100% rename from modules/features/aichat/serviceAi/subAiCallLooping-flow.md rename to modules/aichat/serviceAi/subAiCallLooping-flow.md diff --git a/modules/features/aichat/serviceAi/subAiCallLooping.py b/modules/aichat/serviceAi/subAiCallLooping.py similarity index 100% rename from modules/features/aichat/serviceAi/subAiCallLooping.py rename to modules/aichat/serviceAi/subAiCallLooping.py diff --git a/modules/features/aichat/serviceAi/subContentExtraction.py b/modules/aichat/serviceAi/subContentExtraction.py similarity index 99% rename from modules/features/aichat/serviceAi/subContentExtraction.py rename to modules/aichat/serviceAi/subContentExtraction.py index cace339d..cb6ff743 100644 --- a/modules/features/aichat/serviceAi/subContentExtraction.py +++ b/modules/aichat/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.features.aichat.datamodelFeatureAiChat import ChatDocument +from modules.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/features/aichat/serviceAi/subDocumentIntents.py b/modules/aichat/serviceAi/subDocumentIntents.py similarity index 99% rename from modules/features/aichat/serviceAi/subDocumentIntents.py rename to modules/aichat/serviceAi/subDocumentIntents.py index 2d45179a..a468f1d4 100644 --- a/modules/features/aichat/serviceAi/subDocumentIntents.py +++ b/modules/aichat/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.features.aichat.datamodelFeatureAiChat import ChatDocument +from modules.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/features/aichat/serviceAi/subJsonMerger.py b/modules/aichat/serviceAi/subJsonMerger.py similarity index 100% rename from modules/features/aichat/serviceAi/subJsonMerger.py rename to modules/aichat/serviceAi/subJsonMerger.py diff --git a/modules/features/aichat/serviceAi/subJsonResponseHandling.py b/modules/aichat/serviceAi/subJsonResponseHandling.py similarity index 100% rename from modules/features/aichat/serviceAi/subJsonResponseHandling.py rename to modules/aichat/serviceAi/subJsonResponseHandling.py diff --git a/modules/features/aichat/serviceAi/subLoopingUseCases.py b/modules/aichat/serviceAi/subLoopingUseCases.py similarity index 100% rename from modules/features/aichat/serviceAi/subLoopingUseCases.py rename to modules/aichat/serviceAi/subLoopingUseCases.py diff --git a/modules/features/aichat/serviceAi/subResponseParsing.py b/modules/aichat/serviceAi/subResponseParsing.py similarity index 100% rename from modules/features/aichat/serviceAi/subResponseParsing.py rename to modules/aichat/serviceAi/subResponseParsing.py diff --git a/modules/features/aichat/serviceAi/subStructureFilling.py b/modules/aichat/serviceAi/subStructureFilling.py similarity index 99% rename from modules/features/aichat/serviceAi/subStructureFilling.py rename to modules/aichat/serviceAi/subStructureFilling.py index c5ba2f8a..be2197bb 100644 --- a/modules/features/aichat/serviceAi/subStructureFilling.py +++ b/modules/aichat/serviceAi/subStructureFilling.py @@ -2531,7 +2531,7 @@ CRITICAL: List of accepted section content types (e.g., ["table", "code_block"]) """ try: - from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.aichat.serviceGeneration.renderers.registry import getRenderer # Get renderer for this format renderer = getRenderer(outputFormat, self.services) diff --git a/modules/features/aichat/serviceAi/subStructureGeneration.py b/modules/aichat/serviceAi/subStructureGeneration.py similarity index 99% rename from modules/features/aichat/serviceAi/subStructureGeneration.py rename to modules/aichat/serviceAi/subStructureGeneration.py index ff01c3dd..429a182c 100644 --- a/modules/features/aichat/serviceAi/subStructureGeneration.py +++ b/modules/aichat/serviceAi/subStructureGeneration.py @@ -231,7 +231,7 @@ CRITICAL: raise ValueError("Structure has no documents - cannot generate without documents") # Import renderer registry for format validation (existing infrastructure) - from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.aichat.serviceGeneration.renderers.registry import getRenderer # Validate and fix each document for doc in documents: diff --git a/modules/features/aichat/serviceExtraction/__init__.py b/modules/aichat/serviceExtraction/__init__.py similarity index 100% rename from modules/features/aichat/serviceExtraction/__init__.py rename to modules/aichat/serviceExtraction/__init__.py diff --git a/modules/features/aichat/serviceExtraction/chunking/__init__.py b/modules/aichat/serviceExtraction/chunking/__init__.py similarity index 100% rename from modules/features/aichat/serviceExtraction/chunking/__init__.py rename to modules/aichat/serviceExtraction/chunking/__init__.py diff --git a/modules/features/aichat/serviceExtraction/chunking/chunkerImage.py b/modules/aichat/serviceExtraction/chunking/chunkerImage.py similarity index 100% rename from modules/features/aichat/serviceExtraction/chunking/chunkerImage.py rename to modules/aichat/serviceExtraction/chunking/chunkerImage.py diff --git a/modules/features/aichat/serviceExtraction/chunking/chunkerStructure.py b/modules/aichat/serviceExtraction/chunking/chunkerStructure.py similarity index 100% rename from modules/features/aichat/serviceExtraction/chunking/chunkerStructure.py rename to modules/aichat/serviceExtraction/chunking/chunkerStructure.py diff --git a/modules/features/aichat/serviceExtraction/chunking/chunkerTable.py b/modules/aichat/serviceExtraction/chunking/chunkerTable.py similarity index 100% rename from modules/features/aichat/serviceExtraction/chunking/chunkerTable.py rename to modules/aichat/serviceExtraction/chunking/chunkerTable.py diff --git a/modules/features/aichat/serviceExtraction/chunking/chunkerText.py b/modules/aichat/serviceExtraction/chunking/chunkerText.py similarity index 100% rename from modules/features/aichat/serviceExtraction/chunking/chunkerText.py rename to modules/aichat/serviceExtraction/chunking/chunkerText.py diff --git a/modules/features/aichat/serviceExtraction/extractors/__init__.py b/modules/aichat/serviceExtraction/extractors/__init__.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/__init__.py rename to modules/aichat/serviceExtraction/extractors/__init__.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorBinary.py b/modules/aichat/serviceExtraction/extractors/extractorBinary.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorBinary.py rename to modules/aichat/serviceExtraction/extractors/extractorBinary.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorCsv.py b/modules/aichat/serviceExtraction/extractors/extractorCsv.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorCsv.py rename to modules/aichat/serviceExtraction/extractors/extractorCsv.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorDocx.py b/modules/aichat/serviceExtraction/extractors/extractorDocx.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorDocx.py rename to modules/aichat/serviceExtraction/extractors/extractorDocx.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorHtml.py b/modules/aichat/serviceExtraction/extractors/extractorHtml.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorHtml.py rename to modules/aichat/serviceExtraction/extractors/extractorHtml.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorImage.py b/modules/aichat/serviceExtraction/extractors/extractorImage.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorImage.py rename to modules/aichat/serviceExtraction/extractors/extractorImage.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorJson.py b/modules/aichat/serviceExtraction/extractors/extractorJson.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorJson.py rename to modules/aichat/serviceExtraction/extractors/extractorJson.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorPdf.py b/modules/aichat/serviceExtraction/extractors/extractorPdf.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorPdf.py rename to modules/aichat/serviceExtraction/extractors/extractorPdf.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorPptx.py b/modules/aichat/serviceExtraction/extractors/extractorPptx.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorPptx.py rename to modules/aichat/serviceExtraction/extractors/extractorPptx.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorSql.py b/modules/aichat/serviceExtraction/extractors/extractorSql.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorSql.py rename to modules/aichat/serviceExtraction/extractors/extractorSql.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorText.py b/modules/aichat/serviceExtraction/extractors/extractorText.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorText.py rename to modules/aichat/serviceExtraction/extractors/extractorText.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorXlsx.py b/modules/aichat/serviceExtraction/extractors/extractorXlsx.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorXlsx.py rename to modules/aichat/serviceExtraction/extractors/extractorXlsx.py diff --git a/modules/features/aichat/serviceExtraction/extractors/extractorXml.py b/modules/aichat/serviceExtraction/extractors/extractorXml.py similarity index 100% rename from modules/features/aichat/serviceExtraction/extractors/extractorXml.py rename to modules/aichat/serviceExtraction/extractors/extractorXml.py diff --git a/modules/features/aichat/serviceExtraction/mainServiceExtraction.py b/modules/aichat/serviceExtraction/mainServiceExtraction.py similarity index 99% rename from modules/features/aichat/serviceExtraction/mainServiceExtraction.py rename to modules/aichat/serviceExtraction/mainServiceExtraction.py index abe32352..8777dbee 100644 --- a/modules/features/aichat/serviceExtraction/mainServiceExtraction.py +++ b/modules/aichat/serviceExtraction/mainServiceExtraction.py @@ -11,10 +11,10 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.features.aichat.datamodelFeatureAiChat import ChatDocument +from modules.aichat.datamodelFeatureAiChat import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall -from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.features.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.aichat.aicore.aicoreModelSelector import modelSelector from modules.shared.jsonUtils import stripCodeFences diff --git a/modules/features/aichat/serviceExtraction/merging/__init__.py b/modules/aichat/serviceExtraction/merging/__init__.py similarity index 100% rename from modules/features/aichat/serviceExtraction/merging/__init__.py rename to modules/aichat/serviceExtraction/merging/__init__.py diff --git a/modules/features/aichat/serviceExtraction/merging/mergerDefault.py b/modules/aichat/serviceExtraction/merging/mergerDefault.py similarity index 100% rename from modules/features/aichat/serviceExtraction/merging/mergerDefault.py rename to modules/aichat/serviceExtraction/merging/mergerDefault.py diff --git a/modules/features/aichat/serviceExtraction/merging/mergerTable.py b/modules/aichat/serviceExtraction/merging/mergerTable.py similarity index 100% rename from modules/features/aichat/serviceExtraction/merging/mergerTable.py rename to modules/aichat/serviceExtraction/merging/mergerTable.py diff --git a/modules/features/aichat/serviceExtraction/merging/mergerText.py b/modules/aichat/serviceExtraction/merging/mergerText.py similarity index 100% rename from modules/features/aichat/serviceExtraction/merging/mergerText.py rename to modules/aichat/serviceExtraction/merging/mergerText.py diff --git a/modules/features/aichat/serviceExtraction/subMerger.py b/modules/aichat/serviceExtraction/subMerger.py similarity index 100% rename from modules/features/aichat/serviceExtraction/subMerger.py rename to modules/aichat/serviceExtraction/subMerger.py diff --git a/modules/features/aichat/serviceExtraction/subPipeline.py b/modules/aichat/serviceExtraction/subPipeline.py similarity index 100% rename from modules/features/aichat/serviceExtraction/subPipeline.py rename to modules/aichat/serviceExtraction/subPipeline.py diff --git a/modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py b/modules/aichat/serviceExtraction/subPromptBuilderExtraction.py similarity index 98% rename from modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py rename to modules/aichat/serviceExtraction/subPromptBuilderExtraction.py index 50228983..9b785525 100644 --- a/modules/features/aichat/serviceExtraction/subPromptBuilderExtraction.py +++ b/modules/aichat/serviceExtraction/subPromptBuilderExtraction.py @@ -13,7 +13,7 @@ from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, Operati # Type hint for renderer parameter from typing import TYPE_CHECKING if TYPE_CHECKING: - from modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer + from modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer _RendererLike = BaseRenderer else: _RendererLike = Any diff --git a/modules/features/aichat/serviceExtraction/subRegistry.py b/modules/aichat/serviceExtraction/subRegistry.py similarity index 99% rename from modules/features/aichat/serviceExtraction/subRegistry.py rename to modules/aichat/serviceExtraction/subRegistry.py index 972c1eb7..68c701cc 100644 --- a/modules/features/aichat/serviceExtraction/subRegistry.py +++ b/modules/aichat/serviceExtraction/subRegistry.py @@ -71,7 +71,7 @@ class ExtractorRegistry: module_name = file_path.stem try: # Import the module - module = importlib.import_module(f".{module_name}", package="modules.features.aichat.serviceExtraction.extractors") + module = importlib.import_module(f".{module_name}", package="modules.aichat.serviceExtraction.extractors") # Find all extractor classes in the module for attr_name in dir(module): diff --git a/modules/features/aichat/serviceExtraction/subUtils.py b/modules/aichat/serviceExtraction/subUtils.py similarity index 100% rename from modules/features/aichat/serviceExtraction/subUtils.py rename to modules/aichat/serviceExtraction/subUtils.py diff --git a/modules/features/aichat/serviceGeneration/mainServiceGeneration.py b/modules/aichat/serviceGeneration/mainServiceGeneration.py similarity index 98% rename from modules/features/aichat/serviceGeneration/mainServiceGeneration.py rename to modules/aichat/serviceGeneration/mainServiceGeneration.py index 8aadd081..6953ed7c 100644 --- a/modules/features/aichat/serviceGeneration/mainServiceGeneration.py +++ b/modules/aichat/serviceGeneration/mainServiceGeneration.py @@ -6,8 +6,8 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.features.aichat.datamodelFeatureAiChat import ChatDocument -from modules.features.aichat.serviceGeneration.subDocumentUtility import ( +from modules.aichat.datamodelFeatureAiChat import ChatDocument +from modules.aichat.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, detectMimeTypeFromContent, @@ -414,7 +414,7 @@ class GenerationService: continue # Check output style classification (code/document/image/etc.) from renderer - from modules.features.aichat.serviceGeneration.renderers.registry import getOutputStyle + from modules.aichat.serviceGeneration.renderers.registry import getOutputStyle outputStyle = getOutputStyle(docFormat) if outputStyle: logger.debug(f"Document {doc.get('id', docIndex)} format '{docFormat}' classified as '{outputStyle}' style") @@ -471,8 +471,8 @@ class GenerationService: Complete document structure with populated elements ready for rendering """ try: - from modules.features.aichat.serviceGeneration.subStructureGenerator import StructureGenerator - from modules.features.aichat.serviceGeneration.subContentGenerator import ContentGenerator + from modules.aichat.serviceGeneration.subStructureGenerator import StructureGenerator + from modules.aichat.serviceGeneration.subContentGenerator import ContentGenerator # Phase 1: Generate structure skeleton if progressCallback: @@ -537,7 +537,7 @@ class GenerationService: aiService=None ) -> str: """Get adaptive extraction prompt.""" - from modules.features.aichat.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt + from modules.aichat.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt return await buildExtractionPrompt( outputFormat=outputFormat, userPrompt=userPrompt, diff --git a/modules/features/aichat/serviceGeneration/paths/codePath.py b/modules/aichat/serviceGeneration/paths/codePath.py similarity index 99% rename from modules/features/aichat/serviceGeneration/paths/codePath.py rename to modules/aichat/serviceGeneration/paths/codePath.py index 5e151066..4ab6f79e 100644 --- a/modules/features/aichat/serviceGeneration/paths/codePath.py +++ b/modules/aichat/serviceGeneration/paths/codePath.py @@ -920,7 +920,7 @@ CRITICAL: def _getCodeRenderer(self, fileType: str): """Get code renderer for file type.""" - from modules.features.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.aichat.serviceGeneration.renderers.registry import getRenderer # Map file types to renderer formats formatMap = { diff --git a/modules/features/aichat/serviceGeneration/paths/documentPath.py b/modules/aichat/serviceGeneration/paths/documentPath.py similarity index 100% rename from modules/features/aichat/serviceGeneration/paths/documentPath.py rename to modules/aichat/serviceGeneration/paths/documentPath.py diff --git a/modules/features/aichat/serviceGeneration/paths/imagePath.py b/modules/aichat/serviceGeneration/paths/imagePath.py similarity index 100% rename from modules/features/aichat/serviceGeneration/paths/imagePath.py rename to modules/aichat/serviceGeneration/paths/imagePath.py diff --git a/modules/features/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py b/modules/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py rename to modules/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py diff --git a/modules/features/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py b/modules/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py rename to modules/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py diff --git a/modules/features/aichat/serviceGeneration/renderers/registry.py b/modules/aichat/serviceGeneration/renderers/registry.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/registry.py rename to modules/aichat/serviceGeneration/renderers/registry.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererCodeCsv.py b/modules/aichat/serviceGeneration/renderers/rendererCodeCsv.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererCodeCsv.py rename to modules/aichat/serviceGeneration/renderers/rendererCodeCsv.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererCodeJson.py b/modules/aichat/serviceGeneration/renderers/rendererCodeJson.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererCodeJson.py rename to modules/aichat/serviceGeneration/renderers/rendererCodeJson.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererCodeXml.py b/modules/aichat/serviceGeneration/renderers/rendererCodeXml.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererCodeXml.py rename to modules/aichat/serviceGeneration/renderers/rendererCodeXml.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererCsv.py b/modules/aichat/serviceGeneration/renderers/rendererCsv.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererCsv.py rename to modules/aichat/serviceGeneration/renderers/rendererCsv.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererDocx.py b/modules/aichat/serviceGeneration/renderers/rendererDocx.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererDocx.py rename to modules/aichat/serviceGeneration/renderers/rendererDocx.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererHtml.py b/modules/aichat/serviceGeneration/renderers/rendererHtml.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererHtml.py rename to modules/aichat/serviceGeneration/renderers/rendererHtml.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererImage.py b/modules/aichat/serviceGeneration/renderers/rendererImage.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererImage.py rename to modules/aichat/serviceGeneration/renderers/rendererImage.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererJson.py b/modules/aichat/serviceGeneration/renderers/rendererJson.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererJson.py rename to modules/aichat/serviceGeneration/renderers/rendererJson.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererMarkdown.py b/modules/aichat/serviceGeneration/renderers/rendererMarkdown.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererMarkdown.py rename to modules/aichat/serviceGeneration/renderers/rendererMarkdown.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererPdf.py b/modules/aichat/serviceGeneration/renderers/rendererPdf.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererPdf.py rename to modules/aichat/serviceGeneration/renderers/rendererPdf.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererPptx.py b/modules/aichat/serviceGeneration/renderers/rendererPptx.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererPptx.py rename to modules/aichat/serviceGeneration/renderers/rendererPptx.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererText.py b/modules/aichat/serviceGeneration/renderers/rendererText.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererText.py rename to modules/aichat/serviceGeneration/renderers/rendererText.py diff --git a/modules/features/aichat/serviceGeneration/renderers/rendererXlsx.py b/modules/aichat/serviceGeneration/renderers/rendererXlsx.py similarity index 100% rename from modules/features/aichat/serviceGeneration/renderers/rendererXlsx.py rename to modules/aichat/serviceGeneration/renderers/rendererXlsx.py diff --git a/modules/features/aichat/serviceGeneration/subContentGenerator.py b/modules/aichat/serviceGeneration/subContentGenerator.py similarity index 99% rename from modules/features/aichat/serviceGeneration/subContentGenerator.py rename to modules/aichat/serviceGeneration/subContentGenerator.py index 5995077f..e5c9eec1 100644 --- a/modules/features/aichat/serviceGeneration/subContentGenerator.py +++ b/modules/aichat/serviceGeneration/subContentGenerator.py @@ -12,7 +12,7 @@ import base64 import re import traceback from typing import Dict, Any, Optional, List, Callable -from modules.features.aichat.serviceGeneration.subContentIntegrator import ContentIntegrator +from modules.aichat.serviceGeneration.subContentIntegrator import ContentIntegrator from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/features/aichat/serviceGeneration/subContentIntegrator.py b/modules/aichat/serviceGeneration/subContentIntegrator.py similarity index 100% rename from modules/features/aichat/serviceGeneration/subContentIntegrator.py rename to modules/aichat/serviceGeneration/subContentIntegrator.py diff --git a/modules/features/aichat/serviceGeneration/subDocumentUtility.py b/modules/aichat/serviceGeneration/subDocumentUtility.py similarity index 100% rename from modules/features/aichat/serviceGeneration/subDocumentUtility.py rename to modules/aichat/serviceGeneration/subDocumentUtility.py diff --git a/modules/features/aichat/serviceGeneration/subJsonSchema.py b/modules/aichat/serviceGeneration/subJsonSchema.py similarity index 100% rename from modules/features/aichat/serviceGeneration/subJsonSchema.py rename to modules/aichat/serviceGeneration/subJsonSchema.py diff --git a/modules/features/aichat/serviceGeneration/subPromptBuilderGeneration.py b/modules/aichat/serviceGeneration/subPromptBuilderGeneration.py similarity index 100% rename from modules/features/aichat/serviceGeneration/subPromptBuilderGeneration.py rename to modules/aichat/serviceGeneration/subPromptBuilderGeneration.py diff --git a/modules/features/aichat/serviceGeneration/subStructureGenerator.py b/modules/aichat/serviceGeneration/subStructureGenerator.py similarity index 100% rename from modules/features/aichat/serviceGeneration/subStructureGenerator.py rename to modules/aichat/serviceGeneration/subStructureGenerator.py diff --git a/modules/features/aichat/serviceWeb/mainServiceWeb.py b/modules/aichat/serviceWeb/mainServiceWeb.py similarity index 100% rename from modules/features/aichat/serviceWeb/mainServiceWeb.py rename to modules/aichat/serviceWeb/mainServiceWeb.py diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index 8bac1fd5..aee7d261 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.datamodels.datamodelChat import ActionResult +from modules.aichat.datamodelFeatureAiChat import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 95c93334..f0395a21 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -13,9 +13,9 @@ import logging import json # Import interfaces and models -from modules.features.aichat.interfaceFeatureAiChat import getInterface as getChatInterface +from modules.aichat.interfaceFeatureAiChat import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.features.aichat.datamodelFeatureAiChat import AutomationDefinition, ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.workflows.automation import executeAutomation diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index a82e89dc..ad44f5ee 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -117,7 +117,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -439,7 +439,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.features.aichat.datamodelFeatureAiChat import ChatLog + from modules.aichat.datamodelFeatureAiChat import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/features/neutralizer/interfaceFeatureNeutralizer.py b/modules/features/neutralizer/interfaceFeatureNeutralizer.py new file mode 100644 index 00000000..47439d62 --- /dev/null +++ b/modules/features/neutralizer/interfaceFeatureNeutralizer.py @@ -0,0 +1,176 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Database interface for the Neutralizer feature. +Handles CRUD operations for neutralization configuration and attributes. +""" + +import logging +from typing import Dict, List, Any, Optional + +from modules.features.neutralizer.datamodelFeatureNeutralizer import ( + DataNeutraliserConfig, + DataNeutralizerAttributes, +) +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + + +class InterfaceFeatureNeutralizer: + """Database interface for Neutralizer feature operations""" + + def __init__(self, db, currentUser, mandateId: str, userId: str): + """ + Initialize the interface with database connection and user context. + + Args: + db: Database connection instance + currentUser: Current user object for RBAC + mandateId: Current mandate ID + userId: Current user ID + """ + self.db = db + self.currentUser = currentUser + self.mandateId = mandateId + self.userId = userId + + def getNeutralizationConfig(self) -> Optional[DataNeutraliserConfig]: + """Get the data neutralization configuration for the current user's mandate""" + try: + # Use RBAC filtering + filteredConfigs = getRecordsetWithRBAC( + self.db, + DataNeutraliserConfig, + self.currentUser, + recordFilter={"mandateId": self.mandateId} + ) + + if not filteredConfigs: + return None + + # Filter out database-specific fields + configDict = filteredConfigs[0] + cleanedConfig = {k: v for k, v in configDict.items() if not k.startswith("_")} + return DataNeutraliserConfig(**cleanedConfig) + + except Exception as e: + logger.error(f"Error getting neutralization config: {str(e)}") + return None + + def createOrUpdateNeutralizationConfig( + self, configData: Dict[str, Any] + ) -> DataNeutraliserConfig: + """Create or update the data neutralization configuration""" + try: + # Check if config already exists + existingConfig = self.getNeutralizationConfig() + + if existingConfig: + # Update existing config + updateData = existingConfig.model_dump() + updateData.update(configData) + updateData["updatedAt"] = getUtcTimestamp() + + updatedConfig = DataNeutraliserConfig(**updateData) + self.db.recordModify( + DataNeutraliserConfig, existingConfig.id, updatedConfig + ) + + return updatedConfig + else: + # Create new config + configData["mandateId"] = self.mandateId + configData["userId"] = self.userId + + newConfig = DataNeutraliserConfig(**configData) + createdRecord = self.db.recordCreate(DataNeutraliserConfig, newConfig) + + return DataNeutraliserConfig(**createdRecord) + + except Exception as e: + logger.error(f"Error creating/updating neutralization config: {str(e)}") + raise ValueError(f"Failed to create/update neutralization config: {str(e)}") + + def getNeutralizationAttributes( + self, fileId: Optional[str] = None + ) -> List[DataNeutralizerAttributes]: + """Get neutralization attributes, optionally filtered by file ID""" + try: + filterDict = {"mandateId": self.mandateId} + if fileId: + filterDict["fileId"] = fileId + + # Use RBAC filtering + filteredAttributes = getRecordsetWithRBAC( + self.db, + DataNeutralizerAttributes, + self.currentUser, + recordFilter=filterDict + ) + + # Filter out database-specific fields + cleanedAttributes = [] + for attr in filteredAttributes: + cleanedAttr = {k: v for k, v in attr.items() if not k.startswith("_")} + cleanedAttributes.append(cleanedAttr) + + return [ + DataNeutralizerAttributes(**attr) + for attr in cleanedAttributes + ] + + except Exception as e: + logger.error(f"Error getting neutralization attributes: {str(e)}") + return [] + + def deleteNeutralizationAttributes(self, fileId: str) -> bool: + """Delete all neutralization attributes for a specific file""" + try: + attributes = self.db.getRecordset( + DataNeutralizerAttributes, + recordFilter={"mandateId": self.mandateId, "fileId": fileId}, + ) + + for attribute in attributes: + self.db.recordDelete(DataNeutralizerAttributes, attribute["id"]) + + logger.info( + f"Deleted {len(attributes)} neutralization attributes for file {fileId}" + ) + return True + + except Exception as e: + logger.error(f"Error deleting neutralization attributes: {str(e)}") + return False + + def getAttributeById(self, attributeId: str) -> Optional[Dict[str, Any]]: + """Get a single neutralization attribute by ID""" + try: + attributes = self.db.getRecordset( + DataNeutralizerAttributes, + recordFilter={"mandateId": self.mandateId, "id": attributeId} + ) + if attributes: + return attributes[0] + return None + except Exception as e: + logger.error(f"Error getting attribute by ID: {str(e)}") + return None + + +def getInterface(db, currentUser, mandateId: str, userId: str) -> InterfaceFeatureNeutralizer: + """ + Factory function to create a Neutralizer interface instance. + + Args: + db: Database connection + currentUser: Current user for RBAC + mandateId: Current mandate ID + userId: Current user ID + + Returns: + InterfaceFeatureNeutralizer instance + """ + return InterfaceFeatureNeutralizer(db, currentUser, mandateId, userId) diff --git a/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py index fb47c188..f9e65284 100644 --- a/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py @@ -14,6 +14,7 @@ import json from typing import Dict, List, Any, Optional from modules.features.neutralizer.datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes +from modules.features.neutralizer.interfaceFeatureNeutralizer import InterfaceFeatureNeutralizer # Import all necessary classes and functions for neutralization from .subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute @@ -35,9 +36,19 @@ class NeutralizationService: NamesToParse: List of names to parse and replace (case-insensitive) """ self.services = serviceCenter - self.interfaceDbApp = serviceCenter.interfaceDbApp self.interfaceDbComponent = serviceCenter.interfaceDbComponent + # Create feature-specific interface for neutralizer DB operations + self.interfaceNeutralizer: InterfaceFeatureNeutralizer = None + if serviceCenter and serviceCenter.interfaceDbApp: + dbApp = serviceCenter.interfaceDbApp + self.interfaceNeutralizer = InterfaceFeatureNeutralizer( + db=dbApp.db, + currentUser=dbApp.currentUser, + mandateId=dbApp.mandateId, + userId=dbApp.userId + ) + # Initialize anonymization processors self.NamesToParse = NamesToParse or [] self.textProcessor = TextProcessor(NamesToParse) @@ -47,15 +58,15 @@ class NeutralizationService: def getConfig(self) -> Optional[DataNeutraliserConfig]: """Get the neutralization configuration for the current user's mandate""" - if not self.interfaceDbApp: + if not self.interfaceNeutralizer: return None - return self.interfaceDbApp.getNeutralizationConfig() + return self.interfaceNeutralizer.getNeutralizationConfig() - def saveConfig(self, config_data: Dict[str, Any]) -> DataNeutraliserConfig: + def saveConfig(self, configData: Dict[str, Any]) -> DataNeutraliserConfig: """Save or update the neutralization configuration""" - if not self.interfaceDbApp: + if not self.interfaceNeutralizer: raise ValueError("User context required for saving configuration") - return self.interfaceDbApp.createOrUpdateNeutralizationConfig(config_data) + return self.interfaceNeutralizer.createOrUpdateNeutralizationConfig(configData) # Public API: process text or file @@ -125,44 +136,37 @@ class NeutralizationService: return result def resolveText(self, text: str) -> str: - if not self.interfaceDbApp: + if not self.interfaceNeutralizer: return text try: - placeholder_pattern = r'\[([a-z]+)\.([a-f0-9-]{36})\]' - matches = re.findall(placeholder_pattern, text) - resolved_text = text - for placeholder_type, uid in matches: - attributes = self.interfaceDbApp.db.getRecordset( - DataNeutralizerAttributes, - recordFilter={ - "mandateId": self.interfaceDbApp.mandateId, - "id": uid - } - ) - if attributes: - attribute = attributes[0] - placeholder = f"[{placeholder_type}.{uid}]" - resolved_text = resolved_text.replace(placeholder, attribute["originalText"]) - return resolved_text + placeholderPattern = r'\[([a-z]+)\.([a-f0-9-]{36})\]' + matches = re.findall(placeholderPattern, text) + resolvedText = text + for placeholderType, uid in matches: + attribute = self.interfaceNeutralizer.getAttributeById(uid) + if attribute: + placeholder = f"[{placeholderType}.{uid}]" + resolvedText = resolvedText.replace(placeholder, attribute["originalText"]) + return resolvedText except Exception: return text def getAttributes(self) -> List[DataNeutralizerAttributes]: """Get all neutralization attributes for the current user's mandate""" - if not self.interfaceDbApp: + if not self.interfaceNeutralizer: return [] try: # Use the interface method which properly converts dicts to objects - return self.interfaceDbApp.getNeutralizationAttributes() + return self.interfaceNeutralizer.getNeutralizationAttributes() except Exception as e: logger.error(f"Error getting neutralization attributes: {str(e)}") return [] def deleteNeutralizationAttributes(self, fileId: str) -> bool: """Delete neutralization attributes for a specific file""" - if not self.interfaceDbApp: + if not self.interfaceNeutralizer: return False - return self.interfaceDbApp.deleteNeutralizationAttributes(fileId) + return self.interfaceNeutralizer.deleteNeutralizationAttributes(fileId) def _reloadNamesFromConfig(self) -> None: """Reload names from config and update processors""" diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index e5aa72cc..957221f2 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -10,8 +10,8 @@ import time logger = logging.getLogger(__name__) -from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.features.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.aichat.aicore.aicoreModelSelector import modelSelector from modules.datamodels.datamodelAi import ( AiModel, AiCallOptions, diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 5b6633cb..14a251c7 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -36,10 +36,6 @@ from modules.datamodels.datamodelRbac import ( ) from modules.datamodels.datamodelUam import AccessLevel from modules.datamodels.datamodelSecurity import Token, AuthEvent, TokenStatus -from modules.features.neutralizer.datamodelFeatureNeutralizer import ( - DataNeutraliserConfig, - DataNeutralizerAttributes, -) from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.datamodels.datamodelMembership import ( UserMandate, @@ -2040,115 +2036,6 @@ class AppObjects: logger.error(f"Error during logout: {str(e)}") raise - # Neutralization methods - - def getNeutralizationConfig(self) -> Optional[DataNeutraliserConfig]: - """Get the data neutralization configuration for the current user's mandate""" - try: - # Use RBAC filtering - filtered_configs = getRecordsetWithRBAC(self.db, - DataNeutraliserConfig, - self.currentUser, - recordFilter={"mandateId": self.mandateId} - ) - - if not filtered_configs: - return None - - # Filter out database-specific fields - configDict = filtered_configs[0] - cleanedConfig = {k: v for k, v in configDict.items() if not k.startswith("_")} - return DataNeutraliserConfig(**cleanedConfig) - - except Exception as e: - logger.error(f"Error getting neutralization config: {str(e)}") - return None - - def createOrUpdateNeutralizationConfig( - self, config_data: Dict[str, Any] - ) -> DataNeutraliserConfig: - """Create or update the data neutralization configuration""" - try: - # Check if config already exists - existing_config = self.getNeutralizationConfig() - - if existing_config: - # Update existing config - update_data = existing_config.model_dump() - update_data.update(config_data) - update_data["updatedAt"] = getUtcTimestamp() - - updated_config = DataNeutraliserConfig(**update_data) - self.db.recordModify( - DataNeutraliserConfig, existing_config.id, updated_config - ) - - return updated_config - else: - # Create new config - config_data["mandateId"] = self.mandateId - config_data["userId"] = self.userId - - new_config = DataNeutraliserConfig(**config_data) - created_record = self.db.recordCreate(DataNeutraliserConfig, new_config) - - return DataNeutraliserConfig(**created_record) - - except Exception as e: - logger.error(f"Error creating/updating neutralization config: {str(e)}") - raise ValueError(f"Failed to create/update neutralization config: {str(e)}") - - def getNeutralizationAttributes( - self, file_id: Optional[str] = None - ) -> List[DataNeutralizerAttributes]: - """Get neutralization attributes, optionally filtered by file ID""" - try: - filter_dict = {"mandateId": self.mandateId} - if file_id: - filter_dict["fileId"] = file_id - - # Use RBAC filtering - filtered_attributes = getRecordsetWithRBAC(self.db, - DataNeutralizerAttributes, - self.currentUser, - recordFilter=filter_dict - ) - - # Filter out database-specific fields - cleaned_attributes = [] - for attr in filtered_attributes: - cleanedAttr = {k: v for k, v in attr.items() if not k.startswith("_")} - cleaned_attributes.append(cleanedAttr) - - return [ - DataNeutralizerAttributes(**attr) - for attr in cleaned_attributes - ] - - except Exception as e: - logger.error(f"Error getting neutralization attributes: {str(e)}") - return [] - - def deleteNeutralizationAttributes(self, file_id: str) -> bool: - """Delete all neutralization attributes for a specific file""" - try: - attributes = self.db.getRecordset( - DataNeutralizerAttributes, - recordFilter={"mandateId": self.mandateId, "fileId": file_id}, - ) - - for attribute in attributes: - self.db.recordDelete(DataNeutralizerAttributes, attribute["id"]) - - logger.info( - f"Deleted {len(attributes)} neutralization attributes for file {file_id}" - ) - return True - - except Exception as e: - logger.error(f"Error deleting neutralization attributes: {str(e)}") - return False - # RBAC CRUD Methods def createAccessRule(self, accessRule: AccessRule) -> AccessRule: diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index 9202e448..d1f526c6 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models from feature containers -import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat +import modules.aichat.interfaceFeatureAiChat as interfaceDbChat from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User @@ -76,7 +76,7 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.features.aichat.interfaceFeatureAiChat import getInterface as getChatInterface + from modules.aichat.interfaceFeatureAiChat import getInterface as getChatInterface from modules.interfaces.interfaceDbApp import getRootInterface from modules.workflows.automation import syncAutomationEvents diff --git a/modules/routes/routeDataWorkflows.py b/modules/routes/routeDataWorkflows.py index b4013de1..09bfbabc 100644 --- a/modules/routes/routeDataWorkflows.py +++ b/modules/routes/routeDataWorkflows.py @@ -14,12 +14,12 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces from feature containers -import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat -from modules.features.aichat.interfaceFeatureAiChat import getInterface +import modules.aichat.interfaceFeatureAiChat as interfaceDbChat +from modules.aichat.interfaceFeatureAiChat import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models from feature containers -from modules.features.aichat.datamodelFeatureAiChat import ( +from modules.aichat.datamodelFeatureAiChat import ( ChatWorkflow, ChatMessage, ChatLog, diff --git a/modules/security/passwordUtils.py b/modules/security/passwordUtils.py new file mode 100644 index 00000000..6d6ce235 --- /dev/null +++ b/modules/security/passwordUtils.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Password utility functions for secure password handling. +Uses Argon2 for password hashing. +""" + +from typing import Optional +from passlib.context import CryptContext + +# Password hashing context using Argon2 +_pwdContext = CryptContext(schemes=["argon2"], deprecated="auto") + + +def hashPassword(password: str) -> str: + """ + Hash a password using Argon2. + + Args: + password: Plain text password to hash + + Returns: + Hashed password string + """ + return _pwdContext.hash(password) + + +def verifyPassword(plainPassword: str, hashedPassword: str) -> bool: + """ + Verify a plain password against a hashed password. + + Args: + plainPassword: Plain text password to verify + hashedPassword: Hashed password to compare against + + Returns: + True if password matches, False otherwise + """ + return _pwdContext.verify(plainPassword, hashedPassword) + + +def getPasswordHash(password: Optional[str]) -> Optional[str]: + """ + Hash a password, returning None if password is None. + + Args: + password: Plain text password or None + + Returns: + Hashed password or None if input was None + """ + if password is None: + return None + return _pwdContext.hash(password) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index b5d124ff..ade50b5f 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -19,7 +19,7 @@ import logging from modules.datamodels.datamodelUam import User if TYPE_CHECKING: - from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow + from modules.aichat.datamodelFeatureAiChat import ChatWorkflow logger = logging.getLogger(__name__) diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index ec4cad6b..ef57803d 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.features.aichat.datamodelFeatureAiChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.aichat.datamodelFeatureAiChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index afdad31b..83cb9327 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -161,7 +161,7 @@ class UtilsService: Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChat. """ try: - from modules.features.aichat.interfaceFeatureAiChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.aichat.interfaceFeatureAiChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/shared/__init__.py b/modules/shared/__init__.py new file mode 100644 index 00000000..64b042b8 --- /dev/null +++ b/modules/shared/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Shared utilities module. +Contains common utilities used across the gateway application. +""" + +from . import jsonUtils +from . import timeUtils +from . import attributeUtils +from . import frontendTypes +from . import configuration +from . import eventManagement +from . import auditLogger +from . import debugLogger +from . import progressLogger +from . import callbackRegistry +from . import jsonContinuation +from . import dbMultiTenantOptimizations diff --git a/modules/workflows/automation/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py index 43df551c..332f8d29 100644 --- a/modules/workflows/automation/mainWorkflow.py +++ b/modules/workflows/automation/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index 03f014d5..d6264bed 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult +from modules.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index fecba4e9..ebdada6a 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index dc3231df..4451f0ae 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index 06a4817f..298ecbe4 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index 8dd37872..4dc7c4ef 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult +from modules.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index 0e04dde8..07350a54 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult +from modules.aichat.datamodelFeatureAiChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 8b462e62..753319fe 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index b0cc5a0e..75bf8771 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index dabd2f06..5fd926a1 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index c46c9924..6b4dabe9 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 082497b9..c871fa06 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index c7c94e6c..e299d226 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index c6e2b69a..bd4f666c 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index a33278f3..3b07ca6a 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index a4491efb..b172140b 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 57179415..fd1570f3 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index 60645a06..874867b8 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 899c49d3..8333499f 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index 455592eb..650de4ce 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index b7cf9214..9fb0cc3f 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 3d8ec9d6..69beed2d 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 1fe4992a..c64abf18 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index 0d7c1b6b..4bb1861d 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index 9f08ddae..7dff3f3f 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index 97d42867..e0e2b811 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index 7855627d..418ba477 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index 34aa41e7..9cec8d61 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 88c07269..7942ab21 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index 58d0a2e0..063fd3c5 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index 2c6bde6b..a9b766c1 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index 7059f1d0..13c31219 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index bdabdcbb..469dde0d 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 362c7ba7..bd0618d7 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index b621dbff..2d657596 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.features.aichat.datamodelFeatureAiChat import ActionResult, ActionItem, TaskStep -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionItem, TaskStep +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index 84e0c24f..af9c3f08 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.features.aichat.datamodelFeatureAiChat import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index af2726b7..62f9a72a 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan +from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.features.aichat.datamodelFeatureAiChat import WorkflowModeEnum + from modules.aichat.datamodelFeatureAiChat import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index 085ff694..084024aa 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.features.aichat.datamodelFeatureAiChat import ( +from modules.aichat.datamodelFeatureAiChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index de6016ec..f4533e7d 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskResult, ActionItem -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskResult, ActionItem +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index 606a6ce3..834353b7 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.features.aichat.datamodelFeatureAiChat import ( +from modules.aichat.datamodelFeatureAiChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.features.aichat.datamodelFeatureAiChat import ReviewContext + from modules.aichat.datamodelFeatureAiChat import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.features.aichat.datamodelFeatureAiChat import ReviewResult + from modules.aichat.datamodelFeatureAiChat import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 2094d07a..405aabe1 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.features.aichat.datamodelFeatureAiChat import TaskStep, ActionResult, Observation +from modules.aichat.datamodelFeatureAiChat import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 260ef8b8..b70d632c 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.features.aichat.datamodelFeatureAiChat import Observation + from modules.aichat.datamodelFeatureAiChat import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.features.aichat.datamodelFeatureAiChat import Observation + from modules.aichat.datamodelFeatureAiChat import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict @@ -452,7 +452,7 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.features.aichat.interfaceFeatureAiChat as interfaceDbChat + import modules.aichat.interfaceFeatureAiChat as interfaceDbChat from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() interfaceDbChat = interfaceDbChat.getInterface(rootInterface.currentUser) diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 92432038..e8db412d 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.features.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder +from modules.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index d840cc1e..08007833 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.features.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder +from modules.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index b613d5fd..c8f6b09c 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -7,8 +7,8 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING from modules.datamodels import datamodelChat -from modules.features.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.features.aichat.datamodelFeatureAiChat import ActionDocument + from modules.aichat.datamodelFeatureAiChat import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.features.aichat.datamodelFeatureAiChat import ChatMessage, ChatDocument, ActionDocument + from modules.aichat.datamodelFeatureAiChat import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index a7d28d39..417bb384 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.features.aichat.datamodelFeatureAiChat import ( +from modules.aichat.datamodelFeatureAiChat import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.features.aichat.datamodelFeatureAiChat import TaskContext +from modules.aichat.datamodelFeatureAiChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.features.aichat.datamodelFeatureAiChat import ChatDocument + from modules.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.features.aichat.datamodelFeatureAiChat import ChatDocument + from modules.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.features.aichat.datamodelFeatureAiChat import ActionResult + from modules.aichat.datamodelFeatureAiChat import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/scripts/import_analysis.csv b/scripts/import_analysis.csv index a81312fe..00533906 100644 --- a/scripts/import_analysis.csv +++ b/scripts/import_analysis.csv @@ -27,7 +27,6 @@ gateway.app,modules.routes.routeDataWorkflows,header,Yes gateway.app,modules.routes.routeGdpr,header,Yes gateway.app,modules.routes.routeInvitations,header,Yes gateway.app,modules.routes.routeMessaging,header,Yes -gateway.app,modules.routes.routeOptions,header,Yes gateway.app,modules.routes.routeSecurityAdmin,header,Yes gateway.app,modules.routes.routeSecurityGoogle,header,Yes gateway.app,modules.routes.routeSecurityLocal,header,Yes @@ -42,6 +41,680 @@ gateway.app,os,header,Yes gateway.app,sys,header,Yes gateway.app,unicodedata,header,Yes gateway.app,urllib.parse,header,Yes +gateway.modules.aichat.aicore.aicoreBase,abc,header,Yes +gateway.modules.aichat.aicore.aicoreBase,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicoreBase,time,function getCachedModels,Yes +gateway.modules.aichat.aicore.aicoreBase,typing,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,importlib,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,logging,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelUam,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,modules.security.rbac,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,modules.security.rbacHelpers,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,os,header,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,time,function refreshModels,Yes +gateway.modules.aichat.aicore.aicoreModelRegistry,typing,header,Yes +gateway.modules.aichat.aicore.aicoreModelSelector,logging,header,Yes +gateway.modules.aichat.aicore.aicoreModelSelector,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicoreModelSelector,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,base64,function callAiImage,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,fastapi,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,httpx,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,logging,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,modules.shared.configuration,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,os,header,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,time,function callAiImage,Yes +gateway.modules.aichat.aicore.aicorePluginAnthropic,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginInternal,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicorePluginInternal,logging,header,Yes +gateway.modules.aichat.aicore.aicorePluginInternal,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicorePluginInternal,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,fastapi,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,httpx,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,json,function generateImage,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,logging,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,function generateImage,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,modules.shared.configuration,header,Yes +gateway.modules.aichat.aicore.aicorePluginOpenai,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,fastapi,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,httpx,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webSearch,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,logging,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,function _testConnection,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelTools,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.shared.configuration,header,Yes +gateway.modules.aichat.aicore.aicorePluginPerplexity,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,(relative) .aicoreBase,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,asyncio,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,dataclasses,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,logging,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelTools,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,modules.shared.configuration,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,re,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,re,function _cleanUrl,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,tavily,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,typing,header,Yes +gateway.modules.aichat.aicore.aicorePluginTavily,urllib.parse,function _normalizeUrl,Yes +gateway.modules.aichat.datamodelFeatureAiChat,enum,header,Yes +gateway.modules.aichat.datamodelFeatureAiChat,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes +gateway.modules.aichat.datamodelFeatureAiChat,modules.shared.attributeUtils,header,Yes +gateway.modules.aichat.datamodelFeatureAiChat,modules.shared.timeUtils,header,Yes +gateway.modules.aichat.datamodelFeatureAiChat,pydantic,header,Yes +gateway.modules.aichat.datamodelFeatureAiChat,typing,header,Yes +gateway.modules.aichat.datamodelFeatureAiChat,uuid,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,asyncio,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,datetime,function storeDebugMessageAndDocuments,Yes +gateway.modules.aichat.interfaceFeatureAiChat,json,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,logging,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,math,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.security.rbac,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.configuration,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes +gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.timeUtils,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,os,function storeDebugMessageAndDocuments,Yes +gateway.modules.aichat.interfaceFeatureAiChat,typing,header,Yes +gateway.modules.aichat.interfaceFeatureAiChat,uuid,header,Yes +gateway.modules.aichat.mainAiChat,(relative) .aicore.aicoreModelRegistry,function onStart,Yes +gateway.modules.aichat.mainAiChat,logging,header,Yes +gateway.modules.aichat.mainAiChat,typing,header,Yes +gateway.modules.aichat.routeFeatureAiChat,(relative) .,header,Yes +gateway.modules.aichat.routeFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.routeFeatureAiChat,fastapi,header,Yes +gateway.modules.aichat.routeFeatureAiChat,logging,header,Yes +gateway.modules.aichat.routeFeatureAiChat,modules.auth,header,Yes +gateway.modules.aichat.routeFeatureAiChat,modules.workflows.automation,header,Yes +gateway.modules.aichat.routeFeatureAiChat,typing,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subAiCallLooping,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subContentExtraction,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subDocumentIntents,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subResponseParsing,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subStructureFilling,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subStructureGeneration,function _initializeSubmodules,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,base64,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,json,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,logging,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceExtraction.mainServiceExtraction,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.mainServiceGeneration,function renderResult,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.codePath,function _handleCodeGeneration,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.documentPath,function _handleDocumentGeneration,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.imagePath,function _handleImageGeneration,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.interfaces.interfaceAiObjects,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,re,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,time,header,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,time,function _handleDataExtraction,Yes +gateway.modules.aichat.serviceAi.mainServiceAi,typing,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,(relative) .subLoopingUseCases,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,json,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,logging,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonContinuation,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceAi.subAiCallLooping,typing,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,base64,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,json,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,logging,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function extractTextFromImage,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function processTextContentWithAi,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,function extractAndPrepareContent,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,traceback,function extractTextFromImage,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,traceback,function processTextContentWithAi,Yes +gateway.modules.aichat.serviceAi.subContentExtraction,typing,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,json,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,logging,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,function resolvePreExtractedDocument,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,traceback,function resolvePreExtractedDocument,Yes +gateway.modules.aichat.serviceAi.subDocumentIntents,typing,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,datetime,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,json,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,logging,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,os,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,re,header,Yes +gateway.modules.aichat.serviceAi.subJsonMerger,typing,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,(relative) .subJsonMerger,function mergeJsonStringsWithOverlap,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,json,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,logging,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.debugLogger,function mergeFragmentIntoSection,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function mergeJsonStringsWithOverlap,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _normalizeToElementsStructure,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractRowsFromFragment,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractOverlapAndContinuation,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeWithExplicitOverlap,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _smartConcatenate,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStringsWithOverlapFallback,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStructuresGeneric,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function extractKpiValuesFromIncompleteJson,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,header,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,function _extractRowsFromFragment,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,function _detectAndNormalizeFragment,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,traceback,function _mergeJsonStructuresGeneric,Yes +gateway.modules.aichat.serviceAi.subJsonResponseHandling,typing,header,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,dataclasses,header,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleChapterStructureFinalResult,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeStructureFinalResult,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeContentFinalResult,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,logging,header,Yes +gateway.modules.aichat.serviceAi.subLoopingUseCases,typing,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,json,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,logging,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceAi.subResponseParsing,typing,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,asyncio,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,base64,function _processAiResponseForSection,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,copy,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,json,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,logging,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.aichat.serviceGeneration.renderers.registry,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonContinuation,function buildSectionPromptWithContinuation,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _extractAndMergeMultipleJsonBlocks,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processAiResponseForSection,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processSingleSection,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceAi.subStructureFilling,typing,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,json,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,logging,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.aichat.serviceGeneration.renderers.registry,function generateStructure,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.shared,function generateStructure,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.shared.jsonContinuation,function generateStructure,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceAi.subStructureGeneration,typing,header,Yes +gateway.modules.aichat.serviceExtraction.__init__,(relative) .mainServiceExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,PIL,function chunk,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,base64,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,io,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,typing,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,json,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,typing,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,typing,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerText,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerText,logging,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.chunking.chunkerText,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,base64,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,docx,function _load,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,io,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,bs4,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,PIL,function extract,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,base64,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,io,function extract,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,logging,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorImage,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorJson,json,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorJson,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorJson,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,PyPDF2,function _load,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,base64,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,fitz,function _load,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,io,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,base64,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,io,function extract,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,logging,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,pptx,function _load,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorSql,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorSql,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorText,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorText,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorText,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,datetime,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,io,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,openpyxl,function _load,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXml,typing,header,Yes +gateway.modules.aichat.serviceExtraction.extractors.extractorXml,xml.etree.ElementTree,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerDefault,function applyMerging,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerTable,function applyMerging,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerText,function applyMerging,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subMerger,function applyMerging,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subPipeline,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,asyncio,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,base64,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,json,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,logging,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.aicore.aicoreModelSelector,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,function mergePartResults,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.interfaces.interfaceDbManagement,function extractContent,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.shared.debugLogger,function extractContent,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,time,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,typing,header,Yes +gateway.modules.aichat.serviceExtraction.mainServiceExtraction,uuid,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerDefault,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerDefault,typing,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerTable,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerTable,typing,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerText,(relative) ..subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.merging.mergerText,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subMerger,(relative) .subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.subMerger,logging,header,Yes +gateway.modules.aichat.serviceExtraction.subMerger,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.subMerger,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .mainServiceExtraction,function runExtraction,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .subRegistry,header,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .subUtils,header,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,logging,header,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.subPipeline,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,json,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,logging,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.shared.debugLogger,function buildExtractionPrompt,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerImage,function __init__,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerStructure,function __init__,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerTable,function __init__,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerText,function __init__,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .extractors.extractorBinary,function _auto_discover_extractors,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,importlib,function _auto_discover_extractors,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,logging,header,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,os,function _auto_discover_extractors,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,pathlib,function _auto_discover_extractors,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,traceback,function _auto_discover_extractors,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,traceback,function __init__,Yes +gateway.modules.aichat.serviceExtraction.subRegistry,typing,header,Yes +gateway.modules.aichat.serviceExtraction.subUtils,uuid,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,(relative) .renderers.registry,function _getFormatRenderer,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,base64,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,logging,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceExtraction.subPromptBuilderExtraction,function getAdaptiveExtractionPrompt,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.renderers.registry,function renderReport,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subContentGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subDocumentUtility,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subStructureGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,traceback,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,typing,header,Yes +gateway.modules.aichat.serviceGeneration.mainServiceGeneration,uuid,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,json,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,logging,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.aichat.serviceGeneration.renderers.registry,function _getCodeRenderer,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelDocument,function generateCode,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateCodeStructure,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateSingleFileContent,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonUtils,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,re,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,time,header,Yes +gateway.modules.aichat.serviceGeneration.paths.codePath,typing,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,copy,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,json,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,logging,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,time,header,Yes +gateway.modules.aichat.serviceGeneration.paths.documentPath,typing,header,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,base64,function generateImages,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,json,function generateImages,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,logging,header,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,time,header,Yes +gateway.modules.aichat.serviceGeneration.paths.imagePath,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,abc,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,logging,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,PIL,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,abc,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,datetime,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,json,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,logging,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,function _determineFilename,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,threading,function _getAiStyles,Yes +gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,importlib,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,logging,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,os,function discoverRenderers,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,pathlib,function discoverRenderers,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,sys,function discoverRenderers,Yes +gateway.modules.aichat.serviceGeneration.renderers.registry,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .rendererCsv,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,csv,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .rendererJson,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,json,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,xml.dom,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,xml.etree.ElementTree,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,csv,function _convertRowsToCsv,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,io,function _convertRowsToCsv,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,(relative) .rendererHtml,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,PIL,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,csv,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _setupDocumentStyles,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _createStyle,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.table,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.text,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.ns,function _renderTableFastXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _renderTableFastXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableBordersXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableRowXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _applyHorizontalBordersOnly,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackground,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackgroundFast,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.shared,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,lxml,function _renderTableFastXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,re,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _generateDocxFromJson,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderJsonTable,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderTableFastXml,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,base64,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,base64,function _replaceImageDataUris,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,html,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,html,function _replaceImageDataUris,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,re,function _replaceImageDataUris,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,re,function _extractImages,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,json,function _generateAiImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,logging,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _generateAiImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _compressPromptWithAi,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererImage,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererJson,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererJson,json,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererJson,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,(relative) .rendererHtml,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,PIL,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,base64,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,io,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,json,function _getAiStylesWithPdfColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelAi,function _getAiStylesWithPdfColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,re,function _getAiStylesWithPdfColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,re,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.enums,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.styles,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,function _renderJsonImage,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,datetime,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,json,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,logging,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addTableToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addBulletListToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addHeadingToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addParagraphToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addCodeBlockToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSlideContentWithFrames,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderTextSectionsInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSectionToTextFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addTableToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addBulletListToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addParagraphToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSlideContentWithFrames,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderTextSectionsInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSectionToTextFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addTableToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addHeadingToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addParagraphToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addCodeBlockToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderTextSectionsInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSectionToTextFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlideInFrame,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,re,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,re,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,traceback,function _addImagesToSlide,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererText,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererText,typing,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .rendererCsv,function render,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,base64,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,base64,function _addImageToExcel,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,datetime,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,dateutil,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,io,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,io,function _addImageToExcel,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,json,function _getAiStylesWithExcelColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelAi,function _getAiStylesWithExcelColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.drawing.image,function _addImageToExcel,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.styles,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.utils,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.worksheet.table,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,re,header,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,re,function _getAiStylesWithExcelColors,Yes +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,asyncio,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,base64,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,base64,function _generateImageSection,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,json,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,logging,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.aichat.serviceGeneration.subContentIntegrator,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateSimpleSection,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateImageSection,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.shared.jsonUtils,function _generateSimpleSection,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,re,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,traceback,header,Yes +gateway.modules.aichat.serviceGeneration.subContentGenerator,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subContentIntegrator,json,function integrateContent,Yes +gateway.modules.aichat.serviceGeneration.subContentIntegrator,logging,header,Yes +gateway.modules.aichat.serviceGeneration.subContentIntegrator,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,json,header,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,logging,header,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,os,header,Yes +gateway.modules.aichat.serviceGeneration.subDocumentUtility,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subJsonSchema,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,logging,header,Yes +gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,modules.datamodels.datamodelJson,header,Yes +gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,typing,header,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,json,header,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,json,function _createStructurePrompt,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,logging,header,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelAi,function generateStructure,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelJson,header,Yes +gateway.modules.aichat.serviceGeneration.subStructureGenerator,typing,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,asyncio,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,json,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,logging,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,time,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,time,function _processCrawlResultsWithHierarchy,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,typing,header,Yes +gateway.modules.aichat.serviceWeb.mainServiceWeb,urllib.parse,header,Yes gateway.modules.auth.__init__,(relative) .authentication,header,Yes gateway.modules.auth.__init__,(relative) .csrf,header,Yes gateway.modules.auth.__init__,(relative) .jwtService,header,Yes @@ -246,685 +919,11 @@ gateway.modules.datamodels.datamodelWorkflow,modules.shared.attributeUtils,heade gateway.modules.datamodels.datamodelWorkflow,modules.shared.jsonUtils,header,Yes gateway.modules.datamodels.datamodelWorkflow,pydantic,header,Yes gateway.modules.datamodels.datamodelWorkflow,typing,header,Yes -gateway.modules.datamodels.datamodelWorkflowActions,modules.datamodels.datamodelChat,header,No +gateway.modules.datamodels.datamodelWorkflowActions,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.attributeUtils,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.frontendTypes,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,pydantic,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,typing,header,Yes -gateway.modules.features.aichat.aicore.aicoreBase,abc,header,Yes -gateway.modules.features.aichat.aicore.aicoreBase,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicoreBase,time,function getCachedModels,Yes -gateway.modules.features.aichat.aicore.aicoreBase,typing,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,importlib,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,logging,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.connectors.connectorDbPostgre,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.security.rbac,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,modules.security.rbacHelpers,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,os,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,time,function refreshModels,Yes -gateway.modules.features.aichat.aicore.aicoreModelRegistry,typing,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelSelector,logging,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelSelector,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicoreModelSelector,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,base64,function callAiImage,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,fastapi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,httpx,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,logging,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,modules.shared.configuration,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,os,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,time,function callAiImage,Yes -gateway.modules.features.aichat.aicore.aicorePluginAnthropic,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginInternal,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginInternal,logging,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginInternal,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginInternal,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,fastapi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,httpx,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,json,function generateImage,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,logging,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,function generateImage,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,modules.shared.configuration,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginOpenai,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,fastapi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,httpx,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webSearch,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,logging,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,function _testConnection,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelTools,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,modules.shared.configuration,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginPerplexity,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,(relative) .aicoreBase,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,asyncio,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,dataclasses,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,logging,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelTools,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,modules.shared.configuration,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,re,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,re,function _cleanUrl,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,tavily,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,typing,header,Yes -gateway.modules.features.aichat.aicore.aicorePluginTavily,urllib.parse,function _normalizeUrl,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,enum,header,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,modules.shared.attributeUtils,header,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,modules.shared.timeUtils,header,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,pydantic,header,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,typing,header,Yes -gateway.modules.features.aichat.datamodelFeatureAiChat,uuid,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,asyncio,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,datetime,function storeDebugMessageAndDocuments,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,json,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,logging,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,math,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.connectors.connectorDbPostgre,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelPagination,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelRbac,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceRbac,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.security.rbac,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.security.rootAccess,function setUserContext,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.configuration,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,modules.shared.timeUtils,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,os,function storeDebugMessageAndDocuments,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,typing,header,Yes -gateway.modules.features.aichat.interfaceFeatureAiChat,uuid,header,Yes -gateway.modules.features.aichat.mainAiChat,(relative) .aicore.aicoreModelRegistry,function onStart,Yes -gateway.modules.features.aichat.mainAiChat,logging,header,Yes -gateway.modules.features.aichat.mainAiChat,typing,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,(relative) .,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,fastapi,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,logging,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,modules.auth,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,modules.workflows.automation,header,Yes -gateway.modules.features.aichat.routeFeatureAiChat,typing,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subAiCallLooping,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subContentExtraction,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subDocumentIntents,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subResponseParsing,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subStructureFilling,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,(relative) .subStructureGeneration,function _initializeSubmodules,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,base64,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,json,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,logging,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceExtraction.mainServiceExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.mainServiceGeneration,function renderResult,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.codePath,function _handleCodeGeneration,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.documentPath,function _handleDocumentGeneration,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.features.aichat.serviceGeneration.paths.imagePath,function _handleImageGeneration,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.interfaces.interfaceAiObjects,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,re,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,time,header,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,time,function _handleDataExtraction,Yes -gateway.modules.features.aichat.serviceAi.mainServiceAi,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,(relative) .subLoopingUseCases,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,json,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonContinuation,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceAi.subAiCallLooping,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,base64,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,json,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function extractTextFromImage,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function processTextContentWithAi,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,function extractAndPrepareContent,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,traceback,function extractTextFromImage,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,traceback,function processTextContentWithAi,Yes -gateway.modules.features.aichat.serviceAi.subContentExtraction,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,json,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,function resolvePreExtractedDocument,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,traceback,function resolvePreExtractedDocument,Yes -gateway.modules.features.aichat.serviceAi.subDocumentIntents,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,datetime,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,json,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,os,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,re,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonMerger,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,(relative) .subJsonMerger,function mergeJsonStringsWithOverlap,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,json,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.debugLogger,function mergeFragmentIntoSection,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function mergeJsonStringsWithOverlap,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _normalizeToElementsStructure,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractRowsFromFragment,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractOverlapAndContinuation,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeWithExplicitOverlap,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _smartConcatenate,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStringsWithOverlapFallback,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStructuresGeneric,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function extractKpiValuesFromIncompleteJson,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,header,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,function _extractRowsFromFragment,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,re,function _detectAndNormalizeFragment,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,traceback,function _mergeJsonStructuresGeneric,Yes -gateway.modules.features.aichat.serviceAi.subJsonResponseHandling,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,dataclasses,header,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleChapterStructureFinalResult,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeStructureFinalResult,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeContentFinalResult,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subLoopingUseCases,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,json,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceAi.subResponseParsing,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,asyncio,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,base64,function _processAiResponseForSection,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,copy,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,json,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.features.aichat.serviceGeneration.renderers.registry,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonContinuation,function buildSectionPromptWithContinuation,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _extractAndMergeMultipleJsonBlocks,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processAiResponseForSection,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processSingleSection,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureFilling,typing,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,json,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,logging,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.features.aichat.serviceGeneration.renderers.registry,function generateStructure,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.shared,function generateStructure,No -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.shared.jsonContinuation,function generateStructure,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceAi.subStructureGeneration,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.__init__,(relative) .mainServiceExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,PIL,function chunk,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,io,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerImage,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,json,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerStructure,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerTable,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.chunking.chunkerText,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorBinary,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorCsv,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,docx,function _load,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,io,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorDocx,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,bs4,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorHtml,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,PIL,function extract,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,io,function extract,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorImage,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,json,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorJson,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,PyPDF2,function _load,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,fitz,function _load,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,io,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPdf,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,io,function extract,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,pptx,function _load,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorPptx,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorSql,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorText,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,datetime,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,io,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,openpyxl,function _load,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXlsx,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subRegistry,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.extractors.extractorXml,xml.etree.ElementTree,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerDefault,function applyMerging,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerTable,function applyMerging,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerText,function applyMerging,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subMerger,function applyMerging,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subPipeline,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,(relative) .subRegistry,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,asyncio,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,base64,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,json,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,function mergePartResults,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.aicore.aicoreModelSelector,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.interfaces.interfaceDbManagement,function extractContent,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.shared.debugLogger,function extractContent,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,time,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.mainServiceExtraction,uuid,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerDefault,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerDefault,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerTable,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerText,(relative) ..subUtils,header,No -gateway.modules.features.aichat.serviceExtraction.merging.mergerText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.merging.mergerText,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subMerger,(relative) .subUtils,header,Yes -gateway.modules.features.aichat.serviceExtraction.subMerger,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.subMerger,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.subMerger,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .mainServiceExtraction,function runExtraction,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .subRegistry,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,(relative) .subUtils,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPipeline,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,json,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,modules.shared.debugLogger,function buildExtractionPrompt,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerImage,function __init__,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerStructure,function __init__,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerTable,function __init__,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerText,function __init__,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,(relative) .extractors.extractorBinary,function _auto_discover_extractors,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,importlib,function _auto_discover_extractors,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,logging,header,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,os,function _auto_discover_extractors,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,pathlib,function _auto_discover_extractors,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,traceback,function _auto_discover_extractors,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,traceback,function __init__,Yes -gateway.modules.features.aichat.serviceExtraction.subRegistry,typing,header,Yes -gateway.modules.features.aichat.serviceExtraction.subUtils,uuid,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,(relative) .renderers.registry,function _getFormatRenderer,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceExtraction.subPromptBuilderExtraction,function getAdaptiveExtractionPrompt,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.renderers.registry,function renderReport,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subContentGenerator,function generateDocumentWithTwoPhases,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subDocumentUtility,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,modules.features.aichat.serviceGeneration.subStructureGenerator,function generateDocumentWithTwoPhases,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,traceback,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.mainServiceGeneration,uuid,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelDocument,function generateCode,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.features.aichat.serviceGeneration.renderers.registry,function _getCodeRenderer,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateCodeStructure,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateSingleFileContent,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,modules.shared.jsonUtils,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,time,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.codePath,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,copy,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,time,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.documentPath,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,base64,function generateImages,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,json,function generateImages,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,time,header,Yes -gateway.modules.features.aichat.serviceGeneration.paths.imagePath,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,abc,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,PIL,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,abc,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,datetime,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,function _determineFilename,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,threading,function _getAiStyles,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,importlib,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,os,function discoverRenderers,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,pathlib,function discoverRenderers,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,sys,function discoverRenderers,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.registry,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .rendererCsv,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,csv,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeCsv,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .rendererJson,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeJson,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,xml.dom,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCodeXml,xml.etree.ElementTree,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,csv,function _convertRowsToCsv,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,io,function _convertRowsToCsv,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererCsv,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,(relative) .rendererHtml,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,PIL,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,csv,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _setupDocumentStyles,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _createStyle,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.table,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.text,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.ns,function _renderTableFastXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _renderTableFastXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableBordersXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableRowXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _applyHorizontalBordersOnly,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackground,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackgroundFast,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,docx.shared,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,lxml,function _renderTableFastXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _generateDocxFromJson,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderJsonTable,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderTableFastXml,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererDocx,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,base64,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,base64,function _replaceImageDataUris,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,html,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,html,function _replaceImageDataUris,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,re,function _replaceImageDataUris,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,re,function _extractImages,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererHtml,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,json,function _generateAiImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _generateAiImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _compressPromptWithAi,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererImage,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererJson,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererMarkdown,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,(relative) .rendererHtml,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,PIL,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,base64,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,io,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,json,function _getAiStylesWithPdfColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelAi,function _getAiStylesWithPdfColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,re,function _getAiStylesWithPdfColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,re,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.enums,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.styles,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,function _renderJsonImage,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPdf,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,datetime,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addTableToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addBulletListToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addHeadingToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addParagraphToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addCodeBlockToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSlideContentWithFrames,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderTextSectionsInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSectionToTextFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addTableToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addBulletListToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addParagraphToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSlideContentWithFrames,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderTextSectionsInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSectionToTextFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addTableToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addHeadingToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addParagraphToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addCodeBlockToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderTextSectionsInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSectionToTextFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlideInFrame,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,re,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,traceback,function _addImagesToSlide,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererPptx,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererText,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .rendererCsv,function render,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,base64,function _addImageToExcel,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,datetime,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,dateutil,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,io,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,io,function _addImageToExcel,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,json,function _getAiStylesWithExcelColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelAi,function _getAiStylesWithExcelColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.drawing.image,function _addImageToExcel,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.styles,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.utils,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.worksheet.table,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,re,function _getAiStylesWithExcelColors,Yes -gateway.modules.features.aichat.serviceGeneration.renderers.rendererXlsx,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,asyncio,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,base64,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,base64,function _generateImageSection,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateSimpleSection,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateImageSection,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.features.aichat.serviceGeneration.subContentIntegrator,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.shared.jsonUtils,function _generateSimpleSection,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,re,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,traceback,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentGenerator,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,json,function integrateContent,Yes -gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.subContentIntegrator,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,os,header,Yes -gateway.modules.features.aichat.serviceGeneration.subDocumentUtility,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subJsonSchema,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,modules.datamodels.datamodelJson,header,Yes -gateway.modules.features.aichat.serviceGeneration.subPromptBuilderGeneration,typing,header,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,json,header,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,json,function _createStructurePrompt,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,logging,header,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelAi,function generateStructure,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelJson,header,Yes -gateway.modules.features.aichat.serviceGeneration.subStructureGenerator,typing,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,asyncio,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,json,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,logging,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,modules.datamodels.datamodelAi,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,time,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,time,function _processCrawlResultsWithHierarchy,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,typing,header,Yes -gateway.modules.features.aichat.serviceWeb.mainServiceWeb,urllib.parse,header,Yes gateway.modules.features.automation.mainAutomation,logging,header,Yes gateway.modules.features.automation.mainAutomation,typing,header,Yes gateway.modules.features.automation.routeFeatureAutomation,(relative) .subAutomationTemplates,header,Yes @@ -934,10 +933,10 @@ gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,hea gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,function get_automations,Yes gateway.modules.features.automation.routeFeatureAutomation,json,header,Yes gateway.modules.features.automation.routeFeatureAutomation,logging,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.auth,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.datamodels.datamodelPagination,header,Yes -gateway.modules.features.automation.routeFeatureAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.automation.routeFeatureAutomation,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.services,function execute_automation,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.shared.attributeUtils,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.workflows.automation,header,Yes @@ -993,12 +992,12 @@ gateway.modules.features.chatbot.mainChatbot,asyncio,header,Yes gateway.modules.features.chatbot.mainChatbot,base64,header,Yes gateway.modules.features.chatbot.mainChatbot,json,header,Yes gateway.modules.features.chatbot.mainChatbot,logging,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.features.chatbot.mainChatbot,modules.aichat.datamodelFeatureAiChat,function _emit_log_and_event,Yes gateway.modules.features.chatbot.mainChatbot,modules.connectors.connectorPreprocessor,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelAi,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelDocref,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.chatbot.mainChatbot,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.chatbot.mainChatbot,modules.features.aichat.datamodelFeatureAiChat,function _emit_log_and_event,Yes gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,function _processChatbotMessage,Yes gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.eventManager,header,Yes @@ -1026,9 +1025,6 @@ gateway.modules.features.chatbot.routeFeatureChatbot,modules.interfaces.interfac gateway.modules.features.chatbot.routeFeatureChatbot,modules.shared.timeUtils,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.workflows.automation,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,typing,header,Yes -gateway.modules.features.dynamicOptions.mainDynamicOptions,logging,header,Yes -gateway.modules.features.dynamicOptions.mainDynamicOptions,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.dynamicOptions.mainDynamicOptions,typing,header,Yes gateway.modules.features.featureRegistry,fastapi,header,Yes gateway.modules.features.featureRegistry,glob,header,Yes gateway.modules.features.featureRegistry,importlib,header,Yes @@ -1039,6 +1035,11 @@ gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,modules.shared. gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,pydantic,header,Yes gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,typing,header,Yes gateway.modules.features.neutralizer.datamodelFeatureNeutralizer,uuid,header,Yes +gateway.modules.features.neutralizer.interfaceFeatureNeutralizer,logging,header,Yes +gateway.modules.features.neutralizer.interfaceFeatureNeutralizer,modules.features.neutralizer.datamodelFeatureNeutralizer,header,Yes +gateway.modules.features.neutralizer.interfaceFeatureNeutralizer,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.neutralizer.interfaceFeatureNeutralizer,modules.shared.timeUtils,header,Yes +gateway.modules.features.neutralizer.interfaceFeatureNeutralizer,typing,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,(relative) .datamodelFeatureNeutralizer,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,asyncio,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,logging,header,Yes @@ -1064,6 +1065,7 @@ gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutraliza gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,json,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,logging,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,modules.features.neutralizer.datamodelFeatureNeutralizer,header,Yes +gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,modules.features.neutralizer.interfaceFeatureNeutralizer,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,re,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.mainServiceNeutralization,typing,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.subParseString,(relative) .subPatterns,header,Yes @@ -1099,66 +1101,66 @@ gateway.modules.features.neutralizer.serviceNeutralization.subProcessList,xml.et gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,(relative) .subParseString,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,dataclasses,header,Yes gateway.modules.features.neutralizer.serviceNeutralization.subProcessText,typing,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,enum,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,modules.shared.attributeUtils,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,modules.shared.timeUtils,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,pydantic,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,typing,header,Yes -gateway.modules.features.realEstate.datamodelFeatureRealEstate,uuid,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,logging,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.connectors.connectorDbPostgre,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelRbac,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.interfaces.interfaceRbac,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.security.rbac,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.security.rootAccess,function setUserContext,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,modules.shared.configuration,header,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,re,function _isUUID,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,time,function executeQuery,Yes -gateway.modules.features.realEstate.interfaceFeatureRealEstate,typing,header,Yes -gateway.modules.features.realEstate.mainRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes -gateway.modules.features.realEstate.mainRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes -gateway.modules.features.realEstate.mainRealEstate,fastapi,header,Yes -gateway.modules.features.realEstate.mainRealEstate,json,header,Yes -gateway.modules.features.realEstate.mainRealEstate,logging,header,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,modules.services,header,Yes -gateway.modules.features.realEstate.mainRealEstate,re,function executeIntentBasedOperation,Yes -gateway.modules.features.realEstate.mainRealEstate,shapely.geometry,header,Yes -gateway.modules.features.realEstate.mainRealEstate,shapely.ops,header,Yes -gateway.modules.features.realEstate.mainRealEstate,typing,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,(relative) .mainRealEstate,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,fastapi,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,json,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,logging,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,modules.auth,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,modules.datamodels.datamodelPagination,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,modules.shared.attributeUtils,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,requests,header,Yes -gateway.modules.features.realEstate.routeFeatureRealEstate,typing,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,enum,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,modules.shared.attributeUtils,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,modules.shared.timeUtils,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,pydantic,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,typing,header,Yes +gateway.modules.features.realestate.datamodelFeatureRealEstate,uuid,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,logging,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.security.rbac,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,modules.shared.configuration,header,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,re,function _isUUID,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,time,function executeQuery,Yes +gateway.modules.features.realestate.interfaceFeatureRealEstate,typing,header,Yes +gateway.modules.features.realestate.mainRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realestate.mainRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes +gateway.modules.features.realestate.mainRealEstate,fastapi,header,Yes +gateway.modules.features.realestate.mainRealEstate,json,header,Yes +gateway.modules.features.realestate.mainRealEstate,logging,header,Yes +gateway.modules.features.realestate.mainRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes +gateway.modules.features.realestate.mainRealEstate,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.features.realestate.datamodelFeatureRealEstate,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,modules.services,header,Yes +gateway.modules.features.realestate.mainRealEstate,re,function executeIntentBasedOperation,Yes +gateway.modules.features.realestate.mainRealEstate,shapely.geometry,header,Yes +gateway.modules.features.realestate.mainRealEstate,shapely.ops,header,Yes +gateway.modules.features.realestate.mainRealEstate,typing,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,(relative) .datamodelFeatureRealEstate,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,(relative) .interfaceFeatureRealEstate,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,(relative) .mainRealEstate,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,fastapi,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,json,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,logging,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,modules.auth,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,modules.connectors.connectorSwissTopoMapServer,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,modules.shared.attributeUtils,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,requests,header,Yes +gateway.modules.features.realestate.routeFeatureRealEstate,typing,header,Yes gateway.modules.features.trustee.datamodelFeatureTrustee,modules.shared.attributeUtils,header,Yes gateway.modules.features.trustee.datamodelFeatureTrustee,pydantic,header,Yes gateway.modules.features.trustee.datamodelFeatureTrustee,typing,header,Yes @@ -1201,10 +1203,10 @@ gateway.modules.interfaces.interfaceAiObjects,asyncio,header,Yes gateway.modules.interfaces.interfaceAiObjects,base64,header,Yes gateway.modules.interfaces.interfaceAiObjects,dataclasses,header,Yes gateway.modules.interfaces.interfaceAiObjects,logging,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.aichat.aicore.aicoreModelSelector,header,Yes gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelAi,header,Yes gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.interfaces.interfaceAiObjects,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.modules.interfaces.interfaceAiObjects,modules.features.aichat.aicore.aicoreModelSelector,header,Yes gateway.modules.interfaces.interfaceAiObjects,time,header,Yes gateway.modules.interfaces.interfaceAiObjects,typing,header,Yes gateway.modules.interfaces.interfaceAiObjects,uuid,header,Yes @@ -1229,7 +1231,6 @@ gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelRbac,heade gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelSecurity,header,Yes gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelUam,header,Yes gateway.modules.interfaces.interfaceDbApp,modules.datamodels.datamodelUam,header,Yes -gateway.modules.interfaces.interfaceDbApp,modules.features.neutralizer.datamodelFeatureNeutralizer,header,Yes gateway.modules.interfaces.interfaceDbApp,modules.interfaces.interfaceBootstrap,header,Yes gateway.modules.interfaces.interfaceDbApp,modules.interfaces.interfaceRbac,header,Yes gateway.modules.interfaces.interfaceDbApp,modules.security.rbac,header,Yes @@ -1308,10 +1309,10 @@ gateway.modules.routes.routeAdmin,typing,header,Yes gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes gateway.modules.routes.routeAdminAutomationEvents,logging,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.aichat.interfaceFeatureAiChat,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.auth,header,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeAdminAutomationEvents,modules.features.aichat.interfaceFeatureAiChat,header,Yes -gateway.modules.routes.routeAdminAutomationEvents,modules.features.aichat.interfaceFeatureAiChat,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.interfaces.interfaceDbApp,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.services,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.shared.eventManagement,function get_all_automation_events,Yes @@ -1459,12 +1460,12 @@ gateway.modules.routes.routeDataUsers,typing,header,Yes gateway.modules.routes.routeDataWorkflows,fastapi,header,Yes gateway.modules.routes.routeDataWorkflows,json,header,Yes gateway.modules.routes.routeDataWorkflows,logging,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.routes.routeDataWorkflows,modules.auth,header,Yes gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelPagination,header,Yes gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.features.aichat.interfaceFeatureAiChat,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.routes.routeDataWorkflows,modules.interfaces.interfaceRbac,header,Yes gateway.modules.routes.routeDataWorkflows,modules.services,function get_all_actions,Yes gateway.modules.routes.routeDataWorkflows,modules.services,function get_method_actions,Yes @@ -1513,7 +1514,7 @@ gateway.modules.routes.routeInvitations,modules.datamodels.datamodelRbac,functio gateway.modules.routes.routeInvitations,modules.datamodels.datamodelRbac,function createInvitation,Yes gateway.modules.routes.routeInvitations,modules.datamodels.datamodelUam,header,Yes gateway.modules.routes.routeInvitations,modules.interfaces.interfaceDbApp,header,Yes -gateway.modules.routes.routeInvitations,modules.security.passwordUtils,function registerAndAcceptInvitation,No +gateway.modules.routes.routeInvitations,modules.security.passwordUtils,function registerAndAcceptInvitation,Yes gateway.modules.routes.routeInvitations,modules.shared.configuration,function createInvitation,Yes gateway.modules.routes.routeInvitations,modules.shared.configuration,function listInvitations,Yes gateway.modules.routes.routeInvitations,modules.shared.timeUtils,header,Yes @@ -1532,13 +1533,6 @@ gateway.modules.routes.routeMessaging,modules.interfaces.interfaceDbApp,function gateway.modules.routes.routeMessaging,modules.interfaces.interfaceDbManagement,header,Yes gateway.modules.routes.routeMessaging,modules.services,function triggerSubscription,Yes gateway.modules.routes.routeMessaging,typing,header,Yes -gateway.modules.routes.routeOptions,fastapi,header,Yes -gateway.modules.routes.routeOptions,logging,header,Yes -gateway.modules.routes.routeOptions,modules.auth,header,Yes -gateway.modules.routes.routeOptions,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeOptions,modules.features.dynamicOptions.mainDynamicOptions,header,Yes -gateway.modules.routes.routeOptions,modules.services,header,Yes -gateway.modules.routes.routeOptions,typing,header,Yes gateway.modules.routes.routeSecurityAdmin,fastapi,header,Yes gateway.modules.routes.routeSecurityAdmin,fastapi.responses,header,Yes gateway.modules.routes.routeSecurityAdmin,logging,header,Yes @@ -1633,6 +1627,8 @@ gateway.modules.routes.routeVoiceGoogle,typing,header,Yes gateway.modules.security.__init__,(relative) .rbac,header,Yes gateway.modules.security.__init__,(relative) .rbacHelpers,header,Yes gateway.modules.security.__init__,(relative) .rootAccess,header,Yes +gateway.modules.security.passwordUtils,passlib.context,header,Yes +gateway.modules.security.passwordUtils,typing,header,Yes gateway.modules.security.rbac,logging,header,Yes gateway.modules.security.rbac,modules.connectors.connectorDbPostgre,header,Yes gateway.modules.security.rbac,modules.datamodels.datamodelMembership,header,Yes @@ -1661,18 +1657,18 @@ gateway.modules.services.__init__,(relative) .serviceUtils.mainServiceUtils,func gateway.modules.services.__init__,glob,header,Yes gateway.modules.services.__init__,importlib,header,Yes gateway.modules.services.__init__,logging,header,Yes +gateway.modules.services.__init__,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.services.__init__,modules.datamodels.datamodelUam,header,Yes -gateway.modules.services.__init__,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.services.__init__,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.modules.services.__init__,modules.interfaces.interfaceDbManagement,function __init__,Yes gateway.modules.services.__init__,os,header,Yes gateway.modules.services.__init__,typing,header,Yes gateway.modules.services.serviceChat.mainServiceChat,json,function calculateObjectSize,Yes gateway.modules.services.serviceChat.mainServiceChat,logging,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelAi,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelDocref,function getChatDocumentsFromDocumentList,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelUam,header,Yes -gateway.modules.services.serviceChat.mainServiceChat,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.shared.progressLogger,header,Yes gateway.modules.services.serviceChat.mainServiceChat,sys,function calculateObjectSize,Yes gateway.modules.services.serviceChat.mainServiceChat,typing,header,Yes @@ -1705,8 +1701,8 @@ gateway.modules.services.serviceTicket.mainServiceTicket,modules.interfaces.inte gateway.modules.services.serviceTicket.mainServiceTicket,typing,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,json,function writeDebugArtifact,Yes gateway.modules.services.serviceUtils.mainServiceUtils,logging,header,Yes -gateway.modules.services.serviceUtils.mainServiceUtils,modules.features.aichat.interfaceFeatureAiChat,function storeDebugMessageAndDocuments,Yes -gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared,header,No +gateway.modules.services.serviceUtils.mainServiceUtils,modules.aichat.interfaceFeatureAiChat,function storeDebugMessageAndDocuments,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.configuration,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function writeDebugFile,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function debugLogToFile,Yes @@ -1715,6 +1711,18 @@ gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.eventManag gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.timeUtils,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,re,function sanitizePromptContent,Yes gateway.modules.services.serviceUtils.mainServiceUtils,typing,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes +gateway.modules.shared.__init__,(relative) .,header,Yes gateway.modules.shared.attributeUtils,importlib,header,Yes gateway.modules.shared.attributeUtils,inspect,header,Yes gateway.modules.shared.attributeUtils,logging,header,Yes @@ -1762,9 +1770,6 @@ gateway.modules.shared.eventManagement,apscheduler.triggers.interval,header,Yes gateway.modules.shared.eventManagement,logging,header,Yes gateway.modules.shared.eventManagement,typing,header,Yes gateway.modules.shared.eventManagement,zoneinfo,header,Yes -gateway.modules.shared.frontendOptionsTypes,typing,header,Yes -gateway.modules.shared.frontendOptionsTypes,typing,header,Yes -gateway.modules.shared.frontendOptionsTypes,typing_extensions,header,Yes gateway.modules.shared.frontendTypes,enum,header,Yes gateway.modules.shared.frontendTypes,typing,header,Yes gateway.modules.shared.jsonContinuation,dataclasses,header,Yes @@ -1792,8 +1797,8 @@ gateway.modules.workflows.automation.__init__,(relative) .mainWorkflow,header,Ye gateway.modules.workflows.automation.mainWorkflow,(relative) .subAutomationUtils,header,Yes gateway.modules.workflows.automation.mainWorkflow,json,header,Yes gateway.modules.workflows.automation.mainWorkflow,logging,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.datamodels.datamodelUam,header,Yes -gateway.modules.workflows.automation.mainWorkflow,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.services,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.shared.eventManagement,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.shared.timeUtils,header,Yes @@ -1816,28 +1821,29 @@ gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .summariz gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .translateDocument,header,Yes gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .webResearch,header,Yes gateway.modules.workflows.methods.methodAi.actions.convertDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.convertDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.convertDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.convertDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelAi,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelDocref,function generateCode,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,re,function generateCode,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelAi,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelDocref,function generateDocument,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,re,function generateDocument,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,json,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,logging,header,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes @@ -1845,17 +1851,16 @@ gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.da gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelDocref,function process,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelWorkflow,function process,Yes -gateway.modules.workflows.methods.methodAi.actions.process,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.translateDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.translateDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.translateDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.translateDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.webResearch,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,re,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,typing,header,Yes @@ -1886,9 +1891,9 @@ gateway.modules.workflows.methods.methodBase,typing,header,Yes gateway.modules.workflows.methods.methodChatbot.__init__,(relative) .methodChatbot,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,json,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,logging,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.connectors.connectorPreprocessor,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.datamodels.datamodelDocref,function queryDatabase,Yes -gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.workflows.methods.methodBase,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,time,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,typing,header,Yes @@ -1903,25 +1908,25 @@ gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .get gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .neutralizeData,header,Yes gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .triggerPreprocessingServer,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelDocref,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,time,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,typing,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,json,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,typing,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,logging,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelDocref,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,time,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,typing,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,aiohttp,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,json,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.shared.configuration,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,typing,header,Yes gateway.modules.workflows.methods.methodContext.helpers.documentIndex,datetime,header,Yes @@ -1950,7 +1955,7 @@ gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseC gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseExcelContent,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.shared.configuration,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,uuid,header,Yes @@ -1960,7 +1965,7 @@ gateway.modules.workflows.methods.methodJira.actions.createCsvContent,datetime,h gateway.modules.workflows.methods.methodJira.actions.createCsvContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.createCsvContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,base64,header,Yes @@ -1969,31 +1974,31 @@ gateway.modules.workflows.methods.methodJira.actions.createExcelContent,datetime gateway.modules.workflows.methods.methodJira.actions.createExcelContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.createExcelContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.helpers.adfConverter,logging,header,Yes @@ -2025,30 +2030,30 @@ gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .sen gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,base64,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes -gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.readEmails,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,time,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,logging,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.datamodels.datamodelDocref,function sendDraftEmail,Yes -gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,time,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,typing,header,Yes @@ -2086,55 +2091,55 @@ gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) . gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,datetime,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.datamodels.datamodelDocref,function copyFile,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.datamodels.datamodelDocref,function downloadFileByPath,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,os,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,logging,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,aiohttp,header,Yes gateway.modules.workflows.methods.methodSharepoint.helpers.apiClient,asyncio,header,Yes @@ -2193,23 +2198,23 @@ gateway.modules.workflows.processing.adaptive.progressTracker,datetime,header,Ye gateway.modules.workflows.processing.adaptive.progressTracker,logging,header,Yes gateway.modules.workflows.processing.adaptive.progressTracker,typing,header,Yes gateway.modules.workflows.processing.core.actionExecutor,logging,header,Yes -gateway.modules.workflows.processing.core.actionExecutor,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.actionExecutor,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.core.messageCreator,function _createActionCompletionMessage,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.actionExecutor,time,function executeSingleAction,Yes gateway.modules.workflows.processing.core.actionExecutor,typing,header,Yes gateway.modules.workflows.processing.core.messageCreator,logging,header,Yes -gateway.modules.workflows.processing.core.messageCreator,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.messageCreator,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.core.messageCreator,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.messageCreator,typing,header,Yes gateway.modules.workflows.processing.core.taskPlanner,json,header,Yes gateway.modules.workflows.processing.core.taskPlanner,logging,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.aichat.datamodelFeatureAiChat,function generateTaskPlan,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.datamodels.datamodelAi,header,Yes -gateway.modules.workflows.processing.core.taskPlanner,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.taskPlanner,modules.features.aichat.datamodelFeatureAiChat,function generateTaskPlan,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.promptGenerationTaskplan,header,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.taskPlanner,typing,header,Yes @@ -2218,8 +2223,8 @@ gateway.modules.workflows.processing.core.validator,typing,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,datetime,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeAutomation,json,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,logging,header,Yes -gateway.modules.workflows.processing.modes.modeAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeAutomation,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.shared.timeUtils,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.modes.modeBase,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.shared.stateTools,header,Yes @@ -2228,8 +2233,8 @@ gateway.modules.workflows.processing.modes.modeAutomation,uuid,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeBase,abc,header,Yes gateway.modules.workflows.processing.modes.modeBase,logging,header,Yes -gateway.modules.workflows.processing.modes.modeBase,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeBase,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.actionExecutor,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.messageCreator,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.taskPlanner,header,Yes @@ -2238,6 +2243,10 @@ gateway.modules.workflows.processing.modes.modeBase,typing,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,datetime,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,json,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,logging,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,function _refineDecide,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _actExecute,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _refineDecide,Yes @@ -2246,10 +2255,6 @@ gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamo gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelWorkflow,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelWorkflow,function _actExecute,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,function _refineDecide,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.features.aichat.datamodelFeatureAiChat,function _refineDecide,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _actExecute,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.shared.jsonUtils,function _refineDecide,Yes @@ -2269,7 +2274,7 @@ gateway.modules.workflows.processing.modes.modeDynamic,typing,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.shared.executionState,logging,header,Yes -gateway.modules.workflows.processing.shared.executionState,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.executionState,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.shared.executionState,typing,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,importlib,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,inspect,header,Yes @@ -2279,35 +2284,35 @@ gateway.modules.workflows.processing.shared.methodDiscovery,pkgutil,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,typing,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,json,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,logging,header,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.features.aichat.interfaceFeatureAiChat,function extractLatestRefinementFeedback,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.interfaceFeatureAiChat,function extractLatestRefinementFeedback,Yes gateway.modules.workflows.processing.shared.placeholderFactory,modules.interfaces.interfaceDbApp,function extractLatestRefinementFeedback,Yes gateway.modules.workflows.processing.shared.placeholderFactory,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,typing,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,json,header,Yes -gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.placeholderFactory,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,typing,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,logging,header,Yes -gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.features.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.workflows.processing.shared.placeholderFactory,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,typing,header,Yes gateway.modules.workflows.processing.shared.stateTools,logging,header,Yes gateway.modules.workflows.processing.shared.stateTools,typing,header,Yes gateway.modules.workflows.processing.workflowProcessor,json,header,Yes gateway.modules.workflows.processing.workflowProcessor,logging,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,function fastPathExecute,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,function persistTaskResult,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,function fastPathExecute,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,function fastPathExecute,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.features.aichat.datamodelFeatureAiChat,function persistTaskResult,Yes gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,function initialUnderstanding,Yes gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeAutomation,header,Yes @@ -2322,12 +2327,12 @@ gateway.modules.workflows.processing.workflowProcessor,typing,header,Yes gateway.modules.workflows.workflowManager,asyncio,header,Yes gateway.modules.workflows.workflowManager,json,header,Yes gateway.modules.workflows.workflowManager,logging,header,Yes +gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _executeTasks,Yes +gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes +gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes gateway.modules.workflows.workflowManager,modules.datamodels.datamodelWorkflow,function _executeTasks,Yes -gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _executeTasks,Yes -gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes -gateway.modules.workflows.workflowManager,modules.features.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.placeholderFactory,function _checkIfHistoryAvailable,Yes @@ -2413,11 +2418,11 @@ gateway.tests.conftest,pathlib,header,Yes gateway.tests.conftest,sys,header,Yes gateway.tests.functional.test01_ai_model_selection,asyncio,header,Yes gateway.tests.functional.test01_ai_model_selection,base64,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.aichat.aicore.aicoreModelRegistry,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.aichat.aicore.aicoreModelSelector,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.aichat.serviceAi.mainServiceAi,function initialize,Yes gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.aicore.aicoreModelSelector,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.features.aichat.serviceAi.mainServiceAi,function initialize,Yes gateway.tests.functional.test01_ai_model_selection,modules.interfaces.interfaceAiObjects,function initialize,Yes gateway.tests.functional.test01_ai_model_selection,modules.services,header,Yes gateway.tests.functional.test01_ai_model_selection,os,header,Yes @@ -2433,20 +2438,20 @@ gateway.tests.functional.test02_ai_models,json,header,Yes gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,logging,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function testModel,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function getAllAvailableModels,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicorePluginPerplexity,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicorePluginTavily,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.datamodelFeatureAiChat,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.serviceAi.mainServiceAi,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aichat.serviceExtraction.mainServiceExtraction,function initialize,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function _getTestPromptForOperation,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function getAllAvailableModels,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function testModel,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicoreModelRegistry,function getAllAvailableModels,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicorePluginPerplexity,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.aicore.aicorePluginTavily,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.datamodelFeatureAiChat,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.serviceAi.mainServiceAi,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.features.aichat.serviceExtraction.mainServiceExtraction,function initialize,Yes gateway.tests.functional.test02_ai_models,modules.services,header,Yes gateway.tests.functional.test02_ai_models,modules.shared.configuration,function _testTavilyDirect,Yes gateway.tests.functional.test02_ai_models,os,header,Yes @@ -2459,14 +2464,14 @@ gateway.tests.functional.test03_ai_operations,datetime,header,Yes gateway.tests.functional.test03_ai_operations,json,function printSummary,Yes gateway.tests.functional.test03_ai_operations,json,function testOperation,Yes gateway.tests.functional.test03_ai_operations,logging,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function testOperation,Yes gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function initialize,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.features.aichat.interfaceFeatureAiChat,function testOperation,Yes gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test03_ai_operations,modules.services,function initialize,Yes gateway.tests.functional.test03_ai_operations,modules.workflows.methods.methodAi,function initialize,Yes @@ -2483,11 +2488,11 @@ gateway.tests.functional.test04_ai_behavior,asyncio,header,Yes gateway.tests.functional.test04_ai_behavior,glob,function _getLatestDebugResponse,Yes gateway.tests.functional.test04_ai_behavior,json,header,Yes gateway.tests.functional.test04_ai_behavior,logging,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,modules.aichat.datamodelFeatureAiChat,function initialize,Yes +gateway.tests.functional.test04_ai_behavior,modules.aichat.interfaceFeatureAiChat,function initialize,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelWorkflow,header,Yes -gateway.tests.functional.test04_ai_behavior,modules.features.aichat.datamodelFeatureAiChat,function initialize,Yes -gateway.tests.functional.test04_ai_behavior,modules.features.aichat.interfaceFeatureAiChat,function initialize,Yes gateway.tests.functional.test04_ai_behavior,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test04_ai_behavior,modules.services,header,Yes gateway.tests.functional.test04_ai_behavior,os,header,Yes @@ -2499,9 +2504,9 @@ gateway.tests.functional.test04_ai_behavior,uuid,function initialize,Yes gateway.tests.functional.test05_workflow_with_documents,asyncio,header,Yes gateway.tests.functional.test05_workflow_with_documents,json,header,Yes gateway.tests.functional.test05_workflow_with_documents,logging,function initialize,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test05_workflow_with_documents,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test05_workflow_with_documents,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test05_workflow_with_documents,modules.services,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.workflows.automation,header,Yes @@ -2513,9 +2518,9 @@ gateway.tests.functional.test05_workflow_with_documents,typing,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,asyncio,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,json,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,logging,function initialize,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test06_workflow_prompt_variations,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test06_workflow_prompt_variations,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.services,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.workflows.automation,header,Yes @@ -2528,13 +2533,13 @@ gateway.tests.functional.test06_workflow_prompt_variations,traceback,function te gateway.tests.functional.test06_workflow_prompt_variations,traceback,function runAllTests,Yes gateway.tests.functional.test06_workflow_prompt_variations,typing,header,Yes gateway.tests.functional.test07_json_merge,json,header,Yes -gateway.tests.functional.test07_json_merge,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test07_json_merge,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test07_json_merge,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test07_json_merge,os,header,Yes gateway.tests.functional.test07_json_merge,sys,header,Yes gateway.tests.functional.test07_json_merge,traceback,header,Yes gateway.tests.functional.test08_json_finalization,json,header,Yes -gateway.tests.functional.test08_json_finalization,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test08_json_finalization,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test08_json_finalization,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test08_json_finalization,os,header,Yes gateway.tests.functional.test08_json_finalization,sys,header,Yes @@ -2544,9 +2549,9 @@ gateway.tests.functional.test09_document_generation_formats,asyncio,header,Yes gateway.tests.functional.test09_document_generation_formats,base64,header,Yes gateway.tests.functional.test09_document_generation_formats,json,header,Yes gateway.tests.functional.test09_document_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test09_document_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test09_document_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test09_document_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test09_document_generation_formats,modules.services,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.shared.configuration,function initialize,Yes @@ -2563,9 +2568,9 @@ gateway.tests.functional.test10_document_generation_formats,asyncio,header,Yes gateway.tests.functional.test10_document_generation_formats,base64,header,Yes gateway.tests.functional.test10_document_generation_formats,json,header,Yes gateway.tests.functional.test10_document_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test10_document_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test10_document_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test10_document_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test10_document_generation_formats,modules.services,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.shared.configuration,function initialize,Yes @@ -2582,9 +2587,9 @@ gateway.tests.functional.test11_code_generation_formats,csv,header,Yes gateway.tests.functional.test11_code_generation_formats,io,header,Yes gateway.tests.functional.test11_code_generation_formats,json,header,Yes gateway.tests.functional.test11_code_generation_formats,logging,function initialize,Yes +gateway.tests.functional.test11_code_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.datamodels.datamodelUam,header,Yes -gateway.tests.functional.test11_code_generation_formats,modules.features.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test11_code_generation_formats,modules.features.aichat.interfaceFeatureAiChat,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes gateway.tests.functional.test11_code_generation_formats,modules.services,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.shared.configuration,function initialize,Yes @@ -2598,7 +2603,7 @@ gateway.tests.functional.test11_code_generation_formats,typing,header,Yes gateway.tests.functional.test11_code_generation_formats,xml.etree.ElementTree,header,Yes gateway.tests.functional.test12_json_split_merge,asyncio,header,Yes gateway.tests.functional.test12_json_split_merge,json,header,Yes -gateway.tests.functional.test12_json_split_merge,modules.features.aichat.serviceAi.subJsonMerger,header,Yes +gateway.tests.functional.test12_json_split_merge,modules.aichat.serviceAi.subJsonMerger,header,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonContinuation,header,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function _loadTableJsonExample,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function testJsonSplitMerge,Yes @@ -2627,40 +2632,34 @@ gateway.tests.functional.test14_json_continuation_context,traceback,function tes gateway.tests.functional.test14_json_continuation_context,traceback,function runTest,Yes gateway.tests.functional.test14_json_continuation_context,typing,header,Yes gateway.tests.functional.test_kpi_full,json,header,Yes +gateway.tests.functional.test_kpi_full,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_full,modules.datamodels.datamodelAi,header,Yes -gateway.tests.functional.test_kpi_full,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_full,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test_kpi_full,os,header,Yes gateway.tests.functional.test_kpi_full,pytest,header,Yes gateway.tests.functional.test_kpi_full,sys,header,Yes gateway.tests.functional.test_kpi_incomplete,json,header,Yes +gateway.tests.functional.test_kpi_incomplete,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_incomplete,modules.datamodels.datamodelAi,header,Yes -gateway.tests.functional.test_kpi_incomplete,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_incomplete,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test_kpi_incomplete,os,header,Yes gateway.tests.functional.test_kpi_incomplete,pytest,header,Yes gateway.tests.functional.test_kpi_incomplete,sys,header,Yes gateway.tests.functional.test_kpi_incomplete,traceback,header,Yes gateway.tests.functional.test_kpi_path,json,header,Yes -gateway.tests.functional.test_kpi_path,modules.features.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test_kpi_path,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_path,os,header,Yes gateway.tests.functional.test_kpi_path,sys,header,Yes gateway.tests.functional.test_kpi_path,traceback,header,Yes -gateway.tests.integration.options.test_options_api,app,function app,Yes -gateway.tests.integration.options.test_options_api,fastapi.testclient,header,Yes -gateway.tests.integration.options.test_options_api,modules.datamodels.datamodelUam,header,Yes -gateway.tests.integration.options.test_options_api,modules.interfaces.interfaceDbApp,header,Yes -gateway.tests.integration.options.test_options_api,pytest,header,Yes -gateway.tests.integration.options.test_options_api,secrets,header,Yes gateway.tests.integration.rbac.test_rbac_database,modules.connectors.connectorDbPostgre,header,Yes gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUam,header,Yes gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUam,function testBuildRbacWhereClauseUserConnectionTable,Yes gateway.tests.integration.rbac.test_rbac_database,modules.shared.configuration,header,Yes gateway.tests.integration.rbac.test_rbac_database,pytest,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelDocref,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,function test_extractContentParameters_structure,Yes -gateway.tests.integration.workflows.test_workflow_execution,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.shared.jsonUtils,function test_parseJsonWithModel_with_code_fences,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.shared.jsonUtils,function test_parseJsonWithModel_with_extra_text,Yes gateway.tests.integration.workflows.test_workflow_execution,pytest,header,Yes @@ -2675,12 +2674,6 @@ gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelE gateway.tests.unit.datamodels.test_workflow_models,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.unit.datamodels.test_workflow_models,pytest,header,Yes gateway.tests.unit.datamodels.test_workflow_models,typing,header,Yes -gateway.tests.unit.options.test_frontend_options_types,modules.shared.frontendOptionsTypes,header,Yes -gateway.tests.unit.options.test_frontend_options_types,pytest,header,Yes -gateway.tests.unit.options.test_main_options,modules.datamodels.datamodelUam,header,Yes -gateway.tests.unit.options.test_main_options,modules.features.dynamicOptions.mainDynamicOptions,header,Yes -gateway.tests.unit.options.test_main_options,pytest,header,Yes -gateway.tests.unit.options.test_main_options,unittest.mock,header,Yes gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelRbac,header,Yes gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelUam,header,Yes gateway.tests.unit.rbac.test_rbac_bootstrap,modules.datamodels.datamodelUam,header,Yes @@ -2694,8 +2687,8 @@ gateway.tests.unit.rbac.test_rbac_permissions,modules.security.rbac,header,Yes gateway.tests.unit.rbac.test_rbac_permissions,pytest,header,Yes gateway.tests.unit.rbac.test_rbac_permissions,unittest.mock,header,Yes gateway.tests.unit.services.test_json_extraction_merging,json,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,modules.aichat.serviceExtraction.mainServiceExtraction,header,Yes gateway.tests.unit.services.test_json_extraction_merging,modules.datamodels.datamodelExtraction,header,Yes -gateway.tests.unit.services.test_json_extraction_merging,modules.features.aichat.serviceExtraction.mainServiceExtraction,header,Yes gateway.tests.unit.services.test_json_extraction_merging,os,header,Yes gateway.tests.unit.services.test_json_extraction_merging,sys,header,Yes gateway.tests.unit.services.test_json_extraction_merging,traceback,function main,Yes @@ -2703,13 +2696,13 @@ gateway.tests.unit.utils.test_json_utils,json,header,Yes gateway.tests.unit.utils.test_json_utils,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.unit.utils.test_json_utils,modules.shared.jsonUtils,header,Yes gateway.tests.unit.utils.test_json_utils,pytest,header,Yes +gateway.tests.unit.workflows.test_state_management,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.unit.workflows.test_state_management,modules.datamodels.datamodelWorkflow,header,Yes -gateway.tests.unit.workflows.test_state_management,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.unit.workflows.test_state_management,pytest,header,Yes gateway.tests.unit.workflows.test_state_management,uuid,header,Yes +gateway.tests.validation.test_architecture_validation,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelDocref,header,Yes gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelWorkflow,header,Yes -gateway.tests.validation.test_architecture_validation,modules.features.aichat.datamodelFeatureAiChat,header,Yes gateway.tests.validation.test_architecture_validation,modules.shared.jsonUtils,header,Yes gateway.tests.validation.test_architecture_validation,os,header,Yes gateway.tests.validation.test_architecture_validation,pytest,header,Yes diff --git a/scripts/script_analyze_imports.py b/scripts/script_analyze_imports.py index b6bf9632..c63a556e 100644 --- a/scripts/script_analyze_imports.py +++ b/scripts/script_analyze_imports.py @@ -63,22 +63,42 @@ def checkModuleExists(moduleName: str, currentFile: Path) -> bool: if moduleName.startswith("."): # Relative import - check relative to current file's directory currentDir = currentFile.parent - relativePath = moduleName.lstrip(".") + + # Count the number of dots to determine how many levels up + dotCount = 0 + for char in moduleName: + if char == ".": + dotCount += 1 + else: + break + + # Get the module path after the dots + relativePath = moduleName[dotCount:] + + if not relativePath and dotCount == 1: + return True # "from . import x" - current package + + # Navigate up directories based on dot count + # . = current package (dotCount=1, go to parent dir which is the package) + # .. = parent package (dotCount=2, go up 2 levels) + # etc. + baseDir = currentDir + for _ in range(dotCount - 1): # -1 because currentDir is already at file level + baseDir = baseDir.parent if not relativePath: - return True # "from . import x" - current package + # Pure relative import like "from .. import x" + return baseDir.is_dir() # Convert module path to file path parts = relativePath.split(".") - checkPath = currentDir + checkPath = baseDir for part in parts: checkPath = checkPath / part # Check if it's a package or module if checkPath.is_dir() and (checkPath / "__init__.py").exists(): return True - if (checkPath.parent / f"{checkPath.name}.py").exists(): - return True if checkPath.with_suffix(".py").exists(): return True diff --git a/tests/functional/test01_ai_model_selection.py b/tests/functional/test01_ai_model_selection.py index 0cf36f47..1d4f2d49 100644 --- a/tests/functional/test01_ai_model_selection.py +++ b/tests/functional/test01_ai_model_selection.py @@ -29,8 +29,8 @@ from modules.datamodels.datamodelAi import ( ProcessingModeEnum, ) from modules.datamodels.datamodelUam import User -from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.features.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aichat.aicore.aicoreModelRegistry import modelRegistry +from modules.aichat.aicore.aicoreModelSelector import modelSelector class ModelSelectionTester: @@ -46,7 +46,7 @@ class ModelSelectionTester: self.services = getServices(testUser, None) async def initialize(self) -> None: - from modules.features.aichat.serviceAi.mainServiceAi import AiService + from modules.aichat.serviceAi.mainServiceAi import AiService from modules.interfaces.interfaceAiObjects import AiObjects self.services.ai = await AiService.create(self.services) diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 00953f3e..2f348595 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -68,24 +68,24 @@ class AIModelsTester: logging.getLogger().setLevel(logging.DEBUG) # Initialize the model registry with all connectors - from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry - from modules.features.aichat.aicore.aicorePluginTavily import AiTavily - from modules.features.aichat.aicore.aicorePluginPerplexity import AiPerplexity + from modules.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.aichat.aicore.aicorePluginTavily import AiTavily + from modules.aichat.aicore.aicorePluginPerplexity import AiPerplexity # Note: We don't need to register web connectors for IMAGE_ANALYSE testing # modelRegistry.registerConnector(AiTavily()) # modelRegistry.registerConnector(AiPerplexity()) # The AI service needs to be recreated with proper initialization - from modules.features.aichat.serviceAi.mainServiceAi import AiService + from modules.aichat.serviceAi.mainServiceAi import AiService self.services.ai = await AiService.create(self.services) # Also initialize extraction service for image processing - from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService + from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum + from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( @@ -311,7 +311,7 @@ class AIModelsTester: print(f"{'='*60}") # Get model from registry - from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.aichat.aicore.aicoreModelRegistry import modelRegistry model = modelRegistry.getModel(modelName) if not model: @@ -693,7 +693,7 @@ Width: {crawlWidth} def getAllAvailableModels(self) -> List[Dict[str, Any]]: """Get all available models with their supported operation types.""" - from modules.features.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.aichat.aicore.aicoreModelRegistry import modelRegistry from modules.datamodels.datamodelAi import OperationTypeEnum # Get all models from registry diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index 5875c2bb..de891238 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -94,7 +94,7 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) # Import and initialize services @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.features.aichat.datamodelFeatureAiChat import ChatDocument + from modules.aichat.datamodelFeatureAiChat import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.features.aichat.datamodelFeatureAiChat import ChatMessage + from modules.aichat.datamodelFeatureAiChat import ChatMessage import time testMessage = ChatMessage( @@ -201,7 +201,7 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,7 +283,7 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 6b28439a..4cf21e10 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,10 +42,10 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum + from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat currentTimestamp = time.time() diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 7e6347c2..501204ad 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,10 +20,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class WorkflowWithDocumentsTester: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index ada63dea..e2138a6b 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,10 +22,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class WorkflowPromptVariationsTester: diff --git a/tests/functional/test07_json_merge.py b/tests/functional/test07_json_merge.py index dec51a95..897c6a4f 100644 --- a/tests/functional/test07_json_merge.py +++ b/tests/functional/test07_json_merge.py @@ -11,7 +11,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument # type: ignore diff --git a/tests/functional/test08_json_finalization.py b/tests/functional/test08_json_finalization.py index a6ff570e..f6345150 100644 --- a/tests/functional/test08_json_finalization.py +++ b/tests/functional/test08_json_finalization.py @@ -32,7 +32,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument, extractJsonString, repairBrokenJson # type: ignore diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 9a42c04f..5ecf5a1f 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 1ef6b678..fb1ec6c7 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester10: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index f443ff61..0c58f8ce 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,10 +23,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.features.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.features.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat class CodeGenerationFormatsTester11: diff --git a/tests/functional/test12_json_split_merge.py b/tests/functional/test12_json_split_merge.py index 2632ed2e..ec882824 100644 --- a/tests/functional/test12_json_split_merge.py +++ b/tests/functional/test12_json_split_merge.py @@ -20,7 +20,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import JSON merger from workflow tools -from modules.features.aichat.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger +from modules.aichat.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger from modules.shared.jsonContinuation import getContexts diff --git a/tests/functional/test_kpi_full.py b/tests/functional/test_kpi_full.py index 4457b3e3..c25f48bc 100644 --- a/tests/functional/test_kpi_full.py +++ b/tests/functional/test_kpi_full.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState # Load actual JSON response diff --git a/tests/functional/test_kpi_incomplete.py b/tests/functional/test_kpi_incomplete.py index 90cdcdcb..d7170f48 100644 --- a/tests/functional/test_kpi_incomplete.py +++ b/tests/functional/test_kpi_incomplete.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState from modules.shared.jsonUtils import extractJsonString, repairBrokenJson diff --git a/tests/functional/test_kpi_path.py b/tests/functional/test_kpi_path.py index 66e7293b..824b145c 100644 --- a/tests/functional/test_kpi_path.py +++ b/tests/functional/test_kpi_path.py @@ -10,7 +10,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.features.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # Test JSON matching the actual response test_json = { diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index 26552008..50fafe83 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/services/test_json_extraction_merging.py b/tests/unit/services/test_json_extraction_merging.py index 644b7140..100e3f67 100644 --- a/tests/unit/services/test_json_extraction_merging.py +++ b/tests/unit/services/test_json_extraction_merging.py @@ -14,7 +14,7 @@ import os sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..')) from modules.datamodels.datamodelExtraction import ContentPart -from modules.features.aichat.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService def test_detects_json_with_code_fences(): diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index d649826e..afa873a8 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index dfc46be1..1bde895f 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.features.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.aichat.datamodelFeatureAiChat import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel From 280cafd54a8e2656474c924641afbd4cb7f28d6a Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 23 Jan 2026 01:10:00 +0100 Subject: [PATCH 15/32] refactored features phase II --- app.py | 3 + modules/{aichat => }/aicore/aicoreBase.py | 0 .../aicore/aicoreModelRegistry.py | 0 .../aicore/aicoreModelSelector.py | 0 .../aicore/aicorePluginAnthropic.py | 0 .../aicore/aicorePluginInternal.py | 0 .../{aichat => }/aicore/aicorePluginOpenai.py | 1 - .../aicore/aicorePluginPerplexity.py | 1 - .../{aichat => }/aicore/aicorePluginTavily.py | 0 .../datamodelChat.py} | 37 - .../datamodels/datamodelWorkflowActions.py | 2 +- .../automation/datamodelFeatureAutomation.py | 45 + .../automation/routeFeatureAutomation.py | 5 +- .../chatbot/datamodelFeatureChatbot.py | 37 - .../chatbot/interfaceFeatureChatbot.py | 2 +- modules/features/chatbot/mainChatbot.py | 4 +- .../features/chatbot/routeFeatureChatbot.py | 1 - .../neutralizer/mainNeutralizePlayground.py | 1 - modules/interfaces/interfaceAiObjects.py | 4 +- .../interfaceDbChat.py} | 4 +- modules/routes/routeAdminAutomationEvents.py | 3 +- modules/routes/routeAdminFeatures.py | 2 - modules/routes/routeAdminRbacRoles.py | 2 - .../routeChat.py} | 2 +- modules/routes/routeDataConnections.py | 1 - modules/routes/routeDataMandates.py | 1 - modules/routes/routeDataUsers.py | 5 - modules/routes/routeDataWorkflows.py | 6 +- modules/routes/routeGdpr.py | 3 - modules/routes/routeSecurityGoogle.py | 5 - modules/routes/routeSecurityLocal.py | 2 - modules/routes/routeSecurityMsft.py | 3 - modules/services/__init__.py | 25 +- .../serviceAi}/mainAiChat.py | 2 +- .../serviceAi/mainServiceAi.py | 12 +- .../serviceAi/merge_1.txt | 0 .../serviceAi/subAiCallLooping-flow.md | 0 .../serviceAi/subAiCallLooping.py | 0 .../serviceAi/subContentExtraction.py | 3 +- .../serviceAi/subDocumentIntents.py | 3 +- .../serviceAi/subJsonMerger.py | 0 .../serviceAi/subJsonResponseHandling.py | 13 - .../serviceAi/subLoopingUseCases.py | 0 .../serviceAi/subResponseParsing.py | 0 .../serviceAi/subStructureFilling.py | 2 +- .../serviceAi/subStructureGeneration.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../serviceExtraction/__init__.py | 0 .../serviceExtraction/chunking/__init__.py | 0 .../chunking/chunkerImage.py | 0 .../chunking/chunkerStructure.py | 0 .../chunking/chunkerTable.py | 0 .../serviceExtraction/chunking/chunkerText.py | 0 .../serviceExtraction/extractors/__init__.py | 0 .../extractors/extractorBinary.py | 0 .../extractors/extractorCsv.py | 0 .../extractors/extractorDocx.py | 0 .../extractors/extractorHtml.py | 0 .../extractors/extractorImage.py | 0 .../extractors/extractorJson.py | 0 .../extractors/extractorPdf.py | 0 .../extractors/extractorPptx.py | 0 .../extractors/extractorSql.py | 0 .../extractors/extractorText.py | 0 .../extractors/extractorXlsx.py | 0 .../extractors/extractorXml.py | 0 .../mainServiceExtraction.py | 7 +- .../serviceExtraction/merging/__init__.py | 0 .../merging/mergerDefault.py | 0 .../serviceExtraction/merging/mergerTable.py | 0 .../serviceExtraction/merging/mergerText.py | 0 .../serviceExtraction/subMerger.py | 0 .../serviceExtraction/subPipeline.py | 0 .../subPromptBuilderExtraction.py | 2 +- .../serviceExtraction/subRegistry.py | 2 +- .../serviceExtraction/subUtils.py | 0 .../mainServiceGeneration.py | 12 +- .../serviceGeneration/paths/codePath.py | 2 +- .../serviceGeneration/paths/documentPath.py | 0 .../serviceGeneration/paths/imagePath.py | 0 .../renderers/codeRendererBaseTemplate.py | 0 .../renderers/documentRendererBaseTemplate.py | 1 - .../serviceGeneration/renderers/registry.py | 0 .../renderers/rendererCodeCsv.py | 0 .../renderers/rendererCodeJson.py | 0 .../renderers/rendererCodeXml.py | 0 .../renderers/rendererCsv.py | 0 .../renderers/rendererDocx.py | 0 .../renderers/rendererHtml.py | 0 .../renderers/rendererImage.py | 0 .../renderers/rendererJson.py | 0 .../renderers/rendererMarkdown.py | 0 .../renderers/rendererPdf.py | 0 .../renderers/rendererPptx.py | 0 .../renderers/rendererText.py | 0 .../renderers/rendererXlsx.py | 0 .../serviceGeneration/subContentGenerator.py | 2 +- .../serviceGeneration/subContentIntegrator.py | 0 .../serviceGeneration/subDocumentUtility.py | 0 .../serviceGeneration/subJsonSchema.py | 0 .../subPromptBuilderGeneration.py | 0 .../subStructureGenerator.py | 0 .../services/serviceUtils/mainServiceUtils.py | 2 +- .../serviceWeb/mainServiceWeb.py | 0 modules/workflows/automation/mainWorkflow.py | 3 +- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 4 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 3 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 6 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 6 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 11 +- modules/workflows/workflowManager.py | 7 +- scripts/.$import_diagram.drawio.bkp | 2723 +++++++++++++++++ .../.$import_diagram_containers.drawio.bkp | 317 ++ scripts/function_imports_analysis.txt | 435 +++ scripts/import_analysis.csv | 1623 +++++----- scripts/import_diagram.drawio | 1 + scripts/import_diagram_containers.drawio | 1 + scripts/script_analyze_function_imports.py | 223 ++ scripts/script_generate_container_diagram.py | 221 ++ scripts/script_generate_import_diagram.py | 251 ++ scripts/script_remove_redundant_imports.py | 245 ++ tests/functional/test01_ai_model_selection.py | 6 +- tests/functional/test02_ai_models.py | 16 +- tests/functional/test03_ai_operations.py | 12 +- tests/functional/test04_ai_behavior.py | 4 +- .../test05_workflow_with_documents.py | 4 +- .../test06_workflow_prompt_variations.py | 4 +- tests/functional/test07_json_merge.py | 2 +- tests/functional/test08_json_finalization.py | 2 +- .../test09_document_generation_formats.py | 4 +- .../test10_document_generation_formats.py | 4 +- .../test11_code_generation_formats.py | 4 +- tests/functional/test12_json_split_merge.py | 2 +- tests/functional/test_kpi_full.py | 2 +- tests/functional/test_kpi_incomplete.py | 2 +- tests/functional/test_kpi_path.py | 2 +- .../workflows/test_workflow_execution.py | 2 +- .../services/test_json_extraction_merging.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- 179 files changed, 5422 insertions(+), 1098 deletions(-) rename modules/{aichat => }/aicore/aicoreBase.py (100%) rename modules/{aichat => }/aicore/aicoreModelRegistry.py (100%) rename modules/{aichat => }/aicore/aicoreModelSelector.py (100%) rename modules/{aichat => }/aicore/aicorePluginAnthropic.py (100%) rename modules/{aichat => }/aicore/aicorePluginInternal.py (100%) rename modules/{aichat => }/aicore/aicorePluginOpenai.py (99%) rename modules/{aichat => }/aicore/aicorePluginPerplexity.py (99%) rename modules/{aichat => }/aicore/aicorePluginTavily.py (100%) rename modules/{aichat/datamodelFeatureAiChat.py => datamodels/datamodelChat.py} (92%) create mode 100644 modules/features/automation/datamodelFeatureAutomation.py rename modules/{aichat/interfaceFeatureAiChat.py => interfaces/interfaceDbChat.py} (99%) rename modules/{aichat/routeFeatureAiChat.py => routes/routeChat.py} (97%) rename modules/{aichat => services/serviceAi}/mainAiChat.py (98%) rename modules/{aichat => services}/serviceAi/mainServiceAi.py (98%) rename modules/{aichat => services}/serviceAi/merge_1.txt (100%) rename modules/{aichat => services}/serviceAi/subAiCallLooping-flow.md (100%) rename modules/{aichat => services}/serviceAi/subAiCallLooping.py (100%) rename modules/{aichat => services}/serviceAi/subContentExtraction.py (99%) rename modules/{aichat => services}/serviceAi/subDocumentIntents.py (99%) rename modules/{aichat => services}/serviceAi/subJsonMerger.py (100%) rename modules/{aichat => services}/serviceAi/subJsonResponseHandling.py (99%) rename modules/{aichat => services}/serviceAi/subLoopingUseCases.py (100%) rename modules/{aichat => services}/serviceAi/subResponseParsing.py (100%) rename modules/{aichat => services}/serviceAi/subStructureFilling.py (99%) rename modules/{aichat => services}/serviceAi/subStructureGeneration.py (99%) rename modules/{aichat => services}/serviceExtraction/__init__.py (100%) rename modules/{aichat => services}/serviceExtraction/chunking/__init__.py (100%) rename modules/{aichat => services}/serviceExtraction/chunking/chunkerImage.py (100%) rename modules/{aichat => services}/serviceExtraction/chunking/chunkerStructure.py (100%) rename modules/{aichat => services}/serviceExtraction/chunking/chunkerTable.py (100%) rename modules/{aichat => services}/serviceExtraction/chunking/chunkerText.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/__init__.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorBinary.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorCsv.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorDocx.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorHtml.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorImage.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorJson.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorPdf.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorPptx.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorSql.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorText.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorXlsx.py (100%) rename modules/{aichat => services}/serviceExtraction/extractors/extractorXml.py (100%) rename modules/{aichat => services}/serviceExtraction/mainServiceExtraction.py (99%) rename modules/{aichat => services}/serviceExtraction/merging/__init__.py (100%) rename modules/{aichat => services}/serviceExtraction/merging/mergerDefault.py (100%) rename modules/{aichat => services}/serviceExtraction/merging/mergerTable.py (100%) rename modules/{aichat => services}/serviceExtraction/merging/mergerText.py (100%) rename modules/{aichat => services}/serviceExtraction/subMerger.py (100%) rename modules/{aichat => services}/serviceExtraction/subPipeline.py (100%) rename modules/{aichat => services}/serviceExtraction/subPromptBuilderExtraction.py (98%) rename modules/{aichat => services}/serviceExtraction/subRegistry.py (99%) rename modules/{aichat => services}/serviceExtraction/subUtils.py (100%) rename modules/{aichat => services}/serviceGeneration/mainServiceGeneration.py (98%) rename modules/{aichat => services}/serviceGeneration/paths/codePath.py (99%) rename modules/{aichat => services}/serviceGeneration/paths/documentPath.py (100%) rename modules/{aichat => services}/serviceGeneration/paths/imagePath.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/codeRendererBaseTemplate.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/documentRendererBaseTemplate.py (99%) rename modules/{aichat => services}/serviceGeneration/renderers/registry.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererCodeCsv.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererCodeJson.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererCodeXml.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererCsv.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererDocx.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererHtml.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererImage.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererJson.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererMarkdown.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererPdf.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererPptx.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererText.py (100%) rename modules/{aichat => services}/serviceGeneration/renderers/rendererXlsx.py (100%) rename modules/{aichat => services}/serviceGeneration/subContentGenerator.py (99%) rename modules/{aichat => services}/serviceGeneration/subContentIntegrator.py (100%) rename modules/{aichat => services}/serviceGeneration/subDocumentUtility.py (100%) rename modules/{aichat => services}/serviceGeneration/subJsonSchema.py (100%) rename modules/{aichat => services}/serviceGeneration/subPromptBuilderGeneration.py (100%) rename modules/{aichat => services}/serviceGeneration/subStructureGenerator.py (100%) rename modules/{aichat => services}/serviceWeb/mainServiceWeb.py (100%) create mode 100644 scripts/.$import_diagram.drawio.bkp create mode 100644 scripts/.$import_diagram_containers.drawio.bkp create mode 100644 scripts/function_imports_analysis.txt create mode 100644 scripts/import_diagram.drawio create mode 100644 scripts/import_diagram_containers.drawio create mode 100644 scripts/script_analyze_function_imports.py create mode 100644 scripts/script_generate_container_diagram.py create mode 100644 scripts/script_generate_import_diagram.py create mode 100644 scripts/script_remove_redundant_imports.py diff --git a/app.py b/app.py index 6581065b..4e3e0c67 100644 --- a/app.py +++ b/app.py @@ -485,6 +485,9 @@ app.include_router(rbacAdminExportRouter) from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) +from modules.routes.routeChat import router as chatRouter +app.include_router(chatRouter) + # ============================================================================ # PLUG&PLAY FEATURE ROUTERS # Dynamically load routers from feature containers in modules/features/ diff --git a/modules/aichat/aicore/aicoreBase.py b/modules/aicore/aicoreBase.py similarity index 100% rename from modules/aichat/aicore/aicoreBase.py rename to modules/aicore/aicoreBase.py diff --git a/modules/aichat/aicore/aicoreModelRegistry.py b/modules/aicore/aicoreModelRegistry.py similarity index 100% rename from modules/aichat/aicore/aicoreModelRegistry.py rename to modules/aicore/aicoreModelRegistry.py diff --git a/modules/aichat/aicore/aicoreModelSelector.py b/modules/aicore/aicoreModelSelector.py similarity index 100% rename from modules/aichat/aicore/aicoreModelSelector.py rename to modules/aicore/aicoreModelSelector.py diff --git a/modules/aichat/aicore/aicorePluginAnthropic.py b/modules/aicore/aicorePluginAnthropic.py similarity index 100% rename from modules/aichat/aicore/aicorePluginAnthropic.py rename to modules/aicore/aicorePluginAnthropic.py diff --git a/modules/aichat/aicore/aicorePluginInternal.py b/modules/aicore/aicorePluginInternal.py similarity index 100% rename from modules/aichat/aicore/aicorePluginInternal.py rename to modules/aicore/aicorePluginInternal.py diff --git a/modules/aichat/aicore/aicorePluginOpenai.py b/modules/aicore/aicorePluginOpenai.py similarity index 99% rename from modules/aichat/aicore/aicorePluginOpenai.py rename to modules/aicore/aicorePluginOpenai.py index 711f0c35..c35c6dd6 100644 --- a/modules/aichat/aicore/aicorePluginOpenai.py +++ b/modules/aicore/aicorePluginOpenai.py @@ -298,7 +298,6 @@ class AiOpenai(BaseConnectorAi): promptContent = messages[0]["content"] if messages else "" # Parse prompt using AiCallPromptImage model - from modules.datamodels.datamodelAi import AiCallPromptImage import json try: diff --git a/modules/aichat/aicore/aicorePluginPerplexity.py b/modules/aicore/aicorePluginPerplexity.py similarity index 99% rename from modules/aichat/aicore/aicorePluginPerplexity.py rename to modules/aicore/aicorePluginPerplexity.py index f537d83c..e751f1e9 100644 --- a/modules/aichat/aicore/aicorePluginPerplexity.py +++ b/modules/aicore/aicorePluginPerplexity.py @@ -184,7 +184,6 @@ class AiPerplexity(BaseConnectorAi): ] # Create a model call for testing - from modules.datamodels.datamodelAi import AiCallOptions model = self.getModels()[0] # Get first model for testing testCall = AiModelCall( messages=testMessages, diff --git a/modules/aichat/aicore/aicorePluginTavily.py b/modules/aicore/aicorePluginTavily.py similarity index 100% rename from modules/aichat/aicore/aicorePluginTavily.py rename to modules/aicore/aicorePluginTavily.py diff --git a/modules/aichat/datamodelFeatureAiChat.py b/modules/datamodels/datamodelChat.py similarity index 92% rename from modules/aichat/datamodelFeatureAiChat.py rename to modules/datamodels/datamodelChat.py index 45c5c4eb..c2838ad3 100644 --- a/modules/aichat/datamodelFeatureAiChat.py +++ b/modules/datamodels/datamodelChat.py @@ -1022,40 +1022,3 @@ registerModelLabels( "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, }, ) - - -class AutomationDefinition(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) - schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ - {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, - {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, - {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} - ]}) - template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) - placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) - active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) - eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - - -registerModelLabels( - "AutomationDefinition", - {"en": "Automation Definition", "fr": "Définition d'automatisation"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, - "label": {"en": "Label", "fr": "Libellé"}, - "schedule": {"en": "Schedule", "fr": "Planification"}, - "template": {"en": "Template", "fr": "Modèle"}, - "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, - "active": {"en": "Active", "fr": "Actif"}, - "eventId": {"en": "Event ID", "fr": "ID de l'événement"}, - "status": {"en": "Status", "fr": "Statut"}, - "executionLogs": {"en": "Execution Logs", "fr": "Journaux d'exécution"}, - }, -) diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index aee7d261..8bac1fd5 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.aichat.datamodelFeatureAiChat import ActionResult +from modules.datamodels.datamodelChat import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/automation/datamodelFeatureAutomation.py b/modules/features/automation/datamodelFeatureAutomation.py new file mode 100644 index 00000000..2a774ebd --- /dev/null +++ b/modules/features/automation/datamodelFeatureAutomation.py @@ -0,0 +1,45 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Automation models: AutomationDefinition.""" + +from typing import List, Dict, Any, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +import uuid + + +class AutomationDefinition(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) + schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ + {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, + {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, + {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} + ]}) + template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) + placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) + active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) + eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + + +registerModelLabels( + "AutomationDefinition", + {"en": "Automation Definition", "fr": "Définition d'automatisation"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, + "label": {"en": "Label", "fr": "Libellé"}, + "schedule": {"en": "Schedule", "fr": "Planification"}, + "template": {"en": "Template", "fr": "Modèle"}, + "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, + "active": {"en": "Active", "fr": "Actif"}, + "eventId": {"en": "Event ID", "fr": "ID de l'événement"}, + "status": {"en": "Status", "fr": "Statut"}, + "executionLogs": {"en": "Execution Logs", "fr": "Journaux d'exécution"}, + }, +) diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index f0395a21..49c1606e 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -13,9 +13,10 @@ import logging import json # Import interfaces and models -from modules.aichat.interfaceFeatureAiChat import getInterface as getChatInterface +from modules.interfaces.interfaceDbChat import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.aichat.datamodelFeatureAiChat import AutomationDefinition, ChatWorkflow +from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition +from modules.datamodels.datamodelChat import ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.workflows.automation import executeAutomation diff --git a/modules/features/chatbot/datamodelFeatureChatbot.py b/modules/features/chatbot/datamodelFeatureChatbot.py index 45c5c4eb..c2838ad3 100644 --- a/modules/features/chatbot/datamodelFeatureChatbot.py +++ b/modules/features/chatbot/datamodelFeatureChatbot.py @@ -1022,40 +1022,3 @@ registerModelLabels( "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, }, ) - - -class AutomationDefinition(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) - schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ - {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, - {"value": "0 22 * * *", "label": {"en": "Daily at 22:00", "fr": "Quotidien à 22:00"}}, - {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} - ]}) - template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) - placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) - active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) - eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - executionLogs: List[Dict[str, Any]] = Field(default_factory=list, description="List of execution logs, each containing timestamp, workflowId, status, and messages", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - - -registerModelLabels( - "AutomationDefinition", - {"en": "Automation Definition", "fr": "Définition d'automatisation"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, - "label": {"en": "Label", "fr": "Libellé"}, - "schedule": {"en": "Schedule", "fr": "Planification"}, - "template": {"en": "Template", "fr": "Modèle"}, - "placeholders": {"en": "Placeholders", "fr": "Espaces réservés"}, - "active": {"en": "Active", "fr": "Actif"}, - "eventId": {"en": "Event ID", "fr": "ID de l'événement"}, - "status": {"en": "Status", "fr": "Statut"}, - "executionLogs": {"en": "Execution Logs", "fr": "Journaux d'exécution"}, - }, -) diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index bce7d43e..685a1a4e 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -23,9 +23,9 @@ from .datamodelFeatureChatbot import ( ChatMessage, ChatWorkflow, WorkflowModeEnum, - AutomationDefinition, UserInputRequest ) +from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition import json from modules.datamodels.datamodelUam import User diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index ad44f5ee..560a77b9 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -117,7 +117,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.features.chatbot.datamodelFeatureChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -439,7 +439,6 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.aichat.datamodelFeatureAiChat import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() @@ -1248,7 +1247,6 @@ async def _processChatbotMessage( ) # Retry analysis with empty results context - create NEW analysis with alternative strategies - from modules.features.chatbot.chatbotConstants import get_empty_results_retry_instructions # Build retry prompt with progressively different strategies empty_count = len(sql_queries) diff --git a/modules/features/chatbot/routeFeatureChatbot.py b/modules/features/chatbot/routeFeatureChatbot.py index 3c97c753..ee05e2ac 100644 --- a/modules/features/chatbot/routeFeatureChatbot.py +++ b/modules/features/chatbot/routeFeatureChatbot.py @@ -432,7 +432,6 @@ async def get_chatbot_threads( normalized_workflows.append(normalized_wf) # Create paginated response - from modules.datamodels.datamodelPagination import PaginationMetadata metadata = PaginationMetadata( currentPage=paginationParams.page if paginationParams else 1, pageSize=paginationParams.pageSize if paginationParams else len(workflows), diff --git a/modules/features/neutralizer/mainNeutralizePlayground.py b/modules/features/neutralizer/mainNeutralizePlayground.py index c5932117..bf9aa087 100644 --- a/modules/features/neutralizer/mainNeutralizePlayground.py +++ b/modules/features/neutralizer/mainNeutralizePlayground.py @@ -182,7 +182,6 @@ class SharepointProcessor: async def _getSharepointConnection(self, sharepointPath: str = None): try: - from modules.datamodels.datamodelUam import UserConnection connections = self.services.interfaceDbApp.db.getRecordset( UserConnection, recordFilter={"userId": self.services.interfaceDbApp.userId} diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 957221f2..5c252ff6 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -10,8 +10,8 @@ import time logger = logging.getLogger(__name__) -from modules.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aicore.aicoreModelRegistry import modelRegistry +from modules.aicore.aicoreModelSelector import modelSelector from modules.datamodels.datamodelAi import ( AiModel, AiCallOptions, diff --git a/modules/aichat/interfaceFeatureAiChat.py b/modules/interfaces/interfaceDbChat.py similarity index 99% rename from modules/aichat/interfaceFeatureAiChat.py rename to modules/interfaces/interfaceDbChat.py index de3d42d1..bbe2d9ce 100644 --- a/modules/aichat/interfaceFeatureAiChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -16,16 +16,16 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from .datamodelFeatureAiChat import ( +from modules.datamodels.datamodelChat import ( ChatDocument, ChatStat, ChatLog, ChatMessage, ChatWorkflow, WorkflowModeEnum, - AutomationDefinition, UserInputRequest ) +from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition import json from modules.datamodels.datamodelUam import User diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index d1f526c6..e829f986 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models from feature containers -import modules.aichat.interfaceFeatureAiChat as interfaceDbChat +import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User @@ -76,7 +76,6 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.aichat.interfaceFeatureAiChat import getInterface as getChatInterface from modules.interfaces.interfaceDbApp import getRootInterface from modules.workflows.automation import syncAutomationEvents diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index d62a48e1..d5f1a8f5 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -799,7 +799,6 @@ async def listFeatureInstanceUsers( # Get all FeatureAccess records for this instance from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole from modules.datamodels.datamodelRbac import Role - from modules.datamodels.datamodelUam import UserInDB featureAccesses = rootInterface.db.getRecordset( FeatureAccess, @@ -899,7 +898,6 @@ async def addUserToFeatureInstance( ) # Verify user exists - from modules.datamodels.datamodelUam import UserInDB users = rootInterface.db.getRecordset(UserInDB, recordFilter={"id": data.userId}) if not users: raise HTTPException( diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index c5c45963..70d0beaa 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -364,7 +364,6 @@ async def listUsersWithRoles( # Get all users (SysAdmin sees all) # Use db.getRecordset with UserInDB (the actual database model) - from modules.datamodels.datamodelUam import User, UserInDB allUsersData = interface.db.getRecordset(UserInDB) # Convert to User objects, filtering out sensitive fields users = [] @@ -376,7 +375,6 @@ async def listUsersWithRoles( # Filter by mandate if specified (via UserMandate table) if mandateId: - from modules.datamodels.datamodelMembership import UserMandate userMandates = interface.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) mandateUserIds = {str(um["userId"]) for um in userMandates} users = [u for u in users if str(u.id) in mandateUserIds] diff --git a/modules/aichat/routeFeatureAiChat.py b/modules/routes/routeChat.py similarity index 97% rename from modules/aichat/routeFeatureAiChat.py rename to modules/routes/routeChat.py index 3eaaf624..88d4a4b1 100644 --- a/modules/aichat/routeFeatureAiChat.py +++ b/modules/routes/routeChat.py @@ -16,7 +16,7 @@ from modules.auth import limiter, getRequestContext, RequestContext from . import interfaceFeatureAiChat as interfaceDbChat # Import models -from .datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions from modules.workflows.automation import chatStart, chatStop diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index c7af510e..210e0522 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -182,7 +182,6 @@ async def get_connections( # Perform silent token refresh for expired OAuth connections try: - from modules.auth import token_refresh_service refresh_result = await token_refresh_service.refresh_expired_tokens(currentUser.id) if refresh_result.get("refreshed", 0) > 0: logger.info(f"Silently refreshed {refresh_result['refreshed']} tokens for user {currentUser.id}") diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index ab0ad8c1..37c871ab 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -291,7 +291,6 @@ async def delete_mandate( ) # MULTI-TENANT: Delete all UserMandate entries for this mandate first - from modules.datamodels.datamodelMembership import UserMandate userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) for um in userMandates: appInterface.db.deleteRecord(UserMandate, um["id"]) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 809b3521..ed55b11f 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -252,7 +252,6 @@ async def get_users( elif context.isSysAdmin: # SysAdmin without mandateId sees all users # Get all users directly from database using UserInDB (the actual database model) - from modules.datamodels.datamodelUam import UserInDB allUsers = appInterface.db.getRecordset(UserInDB) # Convert to cleaned dictionaries first for filtering cleanedUsers = [] @@ -378,7 +377,6 @@ async def create_user( appInterface = interfaceDbApp.getInterface(context.user) # Extract fields from request model and call createUser with individual parameters - from modules.datamodels.datamodelUam import AuthAuthority newUser = appInterface.createUser( username=userData.username, password=userData.password, @@ -512,7 +510,6 @@ async def reset_user_password( # SECURITY: Automatically revoke all tokens for the user after password reset try: - from modules.datamodels.datamodelUam import AuthAuthority revoked_count = appInterface.revokeTokensByUser( userId=userId, authority=None, # Revoke all authorities @@ -593,7 +590,6 @@ async def change_password( # SECURITY: Automatically revoke all tokens for the user after password change try: - from modules.datamodels.datamodelUam import AuthAuthority revoked_count = appInterface.revokeTokensByUser( userId=str(context.user.id), authority=None, # Revoke all authorities @@ -654,7 +650,6 @@ async def sendPasswordLink( """ try: from modules.shared.configuration import APP_CONFIG - from modules.interfaces.interfaceDbApp import getRootInterface # Get user interface appInterface = interfaceDbApp.getInterface(context.user) diff --git a/modules/routes/routeDataWorkflows.py b/modules/routes/routeDataWorkflows.py index 09bfbabc..799a9855 100644 --- a/modules/routes/routeDataWorkflows.py +++ b/modules/routes/routeDataWorkflows.py @@ -14,12 +14,12 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces from feature containers -import modules.aichat.interfaceFeatureAiChat as interfaceDbChat -from modules.aichat.interfaceFeatureAiChat import getInterface +import modules.interfaces.interfaceDbChat as interfaceDbChat +from modules.interfaces.interfaceDbChat import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models from feature containers -from modules.aichat.datamodelFeatureAiChat import ( +from modules.datamodels.datamodelChat import ( ChatWorkflow, ChatMessage, ChatLog, diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index f99dcd77..4f5f2aa4 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -121,7 +121,6 @@ async def exportUserData( mandateId = um.get("mandateId") # Get mandate details - from modules.datamodels.datamodelUam import Mandate mandateRecords = rootInterface.db.getRecordset( Mandate, recordFilter={"id": mandateId} @@ -278,7 +277,6 @@ async def exportPortableData( affiliations = [] for um in userMandates: - from modules.datamodels.datamodelUam import Mandate mandateRecords = rootInterface.db.getRecordset( Mandate, recordFilter={"id": um.get("mandateId")} @@ -418,7 +416,6 @@ async def deleteAccount( deletedData.append(f"Tokens deleted: {len(userTokens)}") # 5. Delete user connections (OAuth) - from modules.datamodels.datamodelUam import UserConnection userConnections = rootInterface.db.getRecordset( UserConnection, recordFilter={"userId": str(currentUser.id)} diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 9642df00..39edcf8b 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -170,7 +170,6 @@ async def login( try: if connectionId: rootInterface = getRootInterface() - from modules.datamodels.datamodelUam import UserConnection records = rootInterface.db.getRecordset(UserConnection, recordFilter={"id": connectionId}) if records: record = records[0] @@ -356,7 +355,6 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo # Decode token to get jti for database record from jose import jwt - from modules.auth import SECRET_KEY, ALGORITHM payload = jwt.decode(jwt_token, SECRET_KEY, algorithms=[ALGORITHM]) jti = payload.get("jti") @@ -494,7 +492,6 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo connection.externalEmail = user_info.get("email") # Update connection record directly - from modules.datamodels.datamodelUam import UserConnection rootInterface.db.recordModify(UserConnection, connection_id, connection.model_dump()) @@ -667,7 +664,6 @@ async def verify_token( ) # Get a fresh token via TokenManager convenience method - from modules.auth import TokenManager current_token = TokenManager().getFreshToken(google_connection.id) if not current_token: @@ -741,7 +737,6 @@ async def refresh_token( logger.debug(f"Found Google connection: {google_connection.id}, status={google_connection.status}") # Get the token for this specific connection (fresh if expiring soon) - from modules.auth import TokenManager current_token = TokenManager().getFreshToken(google_connection.id) if not current_token: diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 64984eef..41fe2cab 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -107,7 +107,6 @@ async def login( rootInterface = getRootInterface() # Get default mandate ID - from modules.datamodels.datamodelUam import Mandate defaultMandateId = rootInterface.getInitialId(Mandate) if not defaultMandateId: raise HTTPException( @@ -267,7 +266,6 @@ async def register_user( appInterface = getRootInterface() # Get default mandate ID - from modules.datamodels.datamodelUam import Mandate defaultMandateId = appInterface.getInitialId(Mandate) if not defaultMandateId: raise HTTPException( diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 6d034607..bc637222 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -364,7 +364,6 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo # Decode token to get jti for database record from jose import jwt - from modules.auth import SECRET_KEY, ALGORITHM payload = jwt.decode(jwt_token, SECRET_KEY, algorithms=[ALGORITHM]) jti = payload.get("jti") @@ -726,7 +725,6 @@ async def refresh_token( logger.debug(f"Found Microsoft connection: {msft_connection.id}, status={msft_connection.status}") # Get a fresh token via TokenManager convenience method - from modules.auth import TokenManager current_token = TokenManager().getFreshToken(msft_connection.id) if not current_token: @@ -738,7 +736,6 @@ async def refresh_token( # Always attempt refresh (as per your requirement) - from modules.auth import TokenManager token_manager = TokenManager() refreshedToken = token_manager.refreshToken(current_token) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index ade50b5f..3e0a2560 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -19,7 +19,7 @@ import logging from modules.datamodels.datamodelUam import User if TYPE_CHECKING: - from modules.aichat.datamodelFeatureAiChat import ChatWorkflow + from modules.datamodels.datamodelChat import ChatWorkflow logger = logging.getLogger(__name__) @@ -79,6 +79,12 @@ class Services: self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None + # ============================================================ + # CENTRAL INTERFACE (Chat/Workflow) + # ============================================================ + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) + # ============================================================ # SHARED SERVICES (from modules/services/) # ============================================================ @@ -101,7 +107,22 @@ class Services: self.messaging = PublicService(MessagingService(self)) # ============================================================ - # FEATURE SERVICES (dynamically loaded by filename discovery) + # AI SERVICES (from modules/services/) + # ============================================================ + from .serviceAi.mainServiceAi import AiService + self.ai = PublicService(AiService(self), functionsOnly=False) + + from .serviceExtraction.mainServiceExtraction import ExtractionService + self.extraction = PublicService(ExtractionService(self)) + + from .serviceGeneration.mainServiceGeneration import GenerationService + self.generation = PublicService(GenerationService(self)) + + from .serviceWeb.mainServiceWeb import WebService + self.web = PublicService(WebService(self)) + + # ============================================================ + # FEATURE INTERFACES (dynamically loaded) # ============================================================ self._loadFeatureInterfaces() self._loadFeatureServices() diff --git a/modules/aichat/mainAiChat.py b/modules/services/serviceAi/mainAiChat.py similarity index 98% rename from modules/aichat/mainAiChat.py rename to modules/services/serviceAi/mainAiChat.py index fbd6b91a..2e6514e6 100644 --- a/modules/aichat/mainAiChat.py +++ b/modules/services/serviceAi/mainAiChat.py @@ -154,7 +154,7 @@ async def onStart(eventUser) -> None: Initializes AI connectors for model registry. """ try: - from .aicore.aicoreModelRegistry import modelRegistry + from modules.aicore.aicoreModelRegistry import modelRegistry modelRegistry.ensureConnectorsRegistered() logger.info(f"Feature '{FEATURE_CODE}' started - AI connectors initialized") except Exception as e: diff --git a/modules/aichat/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py similarity index 98% rename from modules/aichat/serviceAi/mainServiceAi.py rename to modules/services/serviceAi/mainServiceAi.py index 40e2e974..296a8032 100644 --- a/modules/aichat/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -6,8 +6,8 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.aichat.datamodelFeatureAiChat import PromptPlaceholder, ChatDocument -from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument +from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData @@ -329,7 +329,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle IMAGE_GENERATE operation type using image generation path.""" - from modules.aichat.serviceGeneration.paths.imagePath import ImageGenerationPath + from modules.services.serviceGeneration.paths.imagePath import ImageGenerationPath imagePath = ImageGenerationPath(self.services) @@ -514,7 +514,7 @@ Respond with ONLY a JSON object in this exact format: ) try: - from modules.aichat.serviceGeneration.mainServiceGeneration import GenerationService + from modules.services.serviceGeneration.mainServiceGeneration import GenerationService generationService = GenerationService(self.services) @@ -829,7 +829,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle code generation using code generation path.""" - from modules.aichat.serviceGeneration.paths.codePath import CodeGenerationPath + from modules.services.serviceGeneration.paths.codePath import CodeGenerationPath codePath = CodeGenerationPath(self.services) return await codePath.generateCode( @@ -852,7 +852,7 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional[str] ) -> AiResponse: """Handle document generation using document generation path.""" - from modules.aichat.serviceGeneration.paths.documentPath import DocumentGenerationPath + from modules.services.serviceGeneration.paths.documentPath import DocumentGenerationPath # Set compression options for document generation options.compressPrompt = False diff --git a/modules/aichat/serviceAi/merge_1.txt b/modules/services/serviceAi/merge_1.txt similarity index 100% rename from modules/aichat/serviceAi/merge_1.txt rename to modules/services/serviceAi/merge_1.txt diff --git a/modules/aichat/serviceAi/subAiCallLooping-flow.md b/modules/services/serviceAi/subAiCallLooping-flow.md similarity index 100% rename from modules/aichat/serviceAi/subAiCallLooping-flow.md rename to modules/services/serviceAi/subAiCallLooping-flow.md diff --git a/modules/aichat/serviceAi/subAiCallLooping.py b/modules/services/serviceAi/subAiCallLooping.py similarity index 100% rename from modules/aichat/serviceAi/subAiCallLooping.py rename to modules/services/serviceAi/subAiCallLooping.py diff --git a/modules/aichat/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py similarity index 99% rename from modules/aichat/serviceAi/subContentExtraction.py rename to modules/services/serviceAi/subContentExtraction.py index cb6ff743..696ba377 100644 --- a/modules/aichat/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.aichat.datamodelFeatureAiChat import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped @@ -387,7 +387,6 @@ class ContentExtractor: ) # Führe Extraktion aus - from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy extractionOptions = ExtractionOptions( prompt=extractionPrompt, diff --git a/modules/aichat/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py similarity index 99% rename from modules/aichat/serviceAi/subDocumentIntents.py rename to modules/services/serviceAi/subDocumentIntents.py index a468f1d4..274a8a5a 100644 --- a/modules/aichat/serviceAi/subDocumentIntents.py +++ b/modules/services/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.aichat.datamodelFeatureAiChat import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped @@ -181,7 +181,6 @@ class DocumentIntentAnalyzer: logger.debug(f"JSON document {document.id} does not have actionType='context.extractContent' (got: {actionType})") if documentData: - from modules.datamodels.datamodelExtraction import ContentExtracted try: # Stelle sicher, dass "id" vorhanden ist diff --git a/modules/aichat/serviceAi/subJsonMerger.py b/modules/services/serviceAi/subJsonMerger.py similarity index 100% rename from modules/aichat/serviceAi/subJsonMerger.py rename to modules/services/serviceAi/subJsonMerger.py diff --git a/modules/aichat/serviceAi/subJsonResponseHandling.py b/modules/services/serviceAi/subJsonResponseHandling.py similarity index 99% rename from modules/aichat/serviceAi/subJsonResponseHandling.py rename to modules/services/serviceAi/subJsonResponseHandling.py index b2cfe084..3adb613c 100644 --- a/modules/aichat/serviceAi/subJsonResponseHandling.py +++ b/modules/services/serviceAi/subJsonResponseHandling.py @@ -1357,7 +1357,6 @@ class JsonResponseHandler: logger.debug(f"Modular merger failed, using fallback: {e}") # Fallback to legacy merger (simplified) - from modules.shared.jsonUtils import normalizeJsonText, stripCodeFences, closeJsonStructures, tryParseJson accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() newFragmentExtracted = stripCodeFences(normalizeJsonText(newFragment)).strip() @@ -1450,7 +1449,6 @@ class JsonResponseHandler: if not jsonString: return None - from modules.shared.jsonUtils import tryParseJson, repairBrokenJson, closeJsonStructures # Try to parse directly first try: @@ -1535,7 +1533,6 @@ class JsonResponseHandler: # Strategy 1: Check if it's an array fragment if jsonStripped.startswith('['): # Try to parse as array - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Close incomplete structures closed = closeJsonStructures(jsonStripped) @@ -1579,7 +1576,6 @@ class JsonResponseHandler: # Strategy 2: Check if it's a partial object (cut mid-structure) # Look for patterns like: {"elements": [...] or {"type": "table"... if jsonStripped.startswith('{'): - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Try to close and parse closed = closeJsonStructures(jsonStripped) @@ -1690,7 +1686,6 @@ class JsonResponseHandler: if not accumulatedElements and not newFragmentElements: # No elements found - try to extract from raw strings # Try to extract any valid JSON structure from raw strings - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Try accumulated first if accumulatedRaw: @@ -2019,7 +2014,6 @@ class JsonResponseHandler: return rows # Pattern 4: Try to parse as JSON array (handles complete arrays) - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Try to close incomplete structures closed = closeJsonStructures(fragmentRaw.strip()) @@ -2292,7 +2286,6 @@ class JsonResponseHandler: if not jsonString: return None, None - from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText, tryParseJson, closeJsonStructures # Extract and normalize JSON extracted = stripCodeFences(normalizeJsonText(jsonString)).strip() @@ -2366,7 +2359,6 @@ class JsonResponseHandler: if not continuationJson: return accumulated - from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText, tryParseJson, closeJsonStructures # Normalize accumulated accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() @@ -2429,7 +2421,6 @@ class JsonResponseHandler: if not jsonString or not jsonString.strip(): return "" - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Strategy 1: Try progressive truncation to find longest valid JSON # Use binary search-like approach for efficiency @@ -2484,7 +2475,6 @@ class JsonResponseHandler: if jsonStripped.startswith('{') or jsonStripped.startswith('['): # Try to extract balanced JSON - from modules.shared.jsonUtils import extractFirstBalancedJson balanced = extractFirstBalancedJson(jsonStripped) if balanced and balanced != jsonStripped: try: @@ -2554,7 +2544,6 @@ class JsonResponseHandler: if not accumulated or not newFragment: return "" - from modules.shared.jsonUtils import closeJsonStructures, tryParseJson # Extract valid JSON prefixes from both accumulatedValid = JsonResponseHandler._extractValidJsonPrefix(accumulated) @@ -2647,7 +2636,6 @@ class JsonResponseHandler: if not newFragment: return accumulated - from modules.shared.jsonUtils import tryParseJson, closeJsonStructures # Strategy 1: Try to extract valid JSON parts from both fragments # This handles random cuts better by finding the longest valid prefix/suffix @@ -2894,7 +2882,6 @@ class JsonResponseHandler: try: # Use existing JSON completion function to close incomplete structures - from modules.shared.jsonUtils import extractJsonString, closeJsonStructures # Extract JSON string and complete it with missing closing elements extracted = extractJsonString(jsonString) diff --git a/modules/aichat/serviceAi/subLoopingUseCases.py b/modules/services/serviceAi/subLoopingUseCases.py similarity index 100% rename from modules/aichat/serviceAi/subLoopingUseCases.py rename to modules/services/serviceAi/subLoopingUseCases.py diff --git a/modules/aichat/serviceAi/subResponseParsing.py b/modules/services/serviceAi/subResponseParsing.py similarity index 100% rename from modules/aichat/serviceAi/subResponseParsing.py rename to modules/services/serviceAi/subResponseParsing.py diff --git a/modules/aichat/serviceAi/subStructureFilling.py b/modules/services/serviceAi/subStructureFilling.py similarity index 99% rename from modules/aichat/serviceAi/subStructureFilling.py rename to modules/services/serviceAi/subStructureFilling.py index be2197bb..5145ad54 100644 --- a/modules/aichat/serviceAi/subStructureFilling.py +++ b/modules/services/serviceAi/subStructureFilling.py @@ -2531,7 +2531,7 @@ CRITICAL: List of accepted section content types (e.g., ["table", "code_block"]) """ try: - from modules.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.services.serviceGeneration.renderers.registry import getRenderer # Get renderer for this format renderer = getRenderer(outputFormat, self.services) diff --git a/modules/aichat/serviceAi/subStructureGeneration.py b/modules/services/serviceAi/subStructureGeneration.py similarity index 99% rename from modules/aichat/serviceAi/subStructureGeneration.py rename to modules/services/serviceAi/subStructureGeneration.py index 429a182c..64624b84 100644 --- a/modules/aichat/serviceAi/subStructureGeneration.py +++ b/modules/services/serviceAi/subStructureGeneration.py @@ -231,7 +231,7 @@ CRITICAL: raise ValueError("Structure has no documents - cannot generate without documents") # Import renderer registry for format validation (existing infrastructure) - from modules.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.services.serviceGeneration.renderers.registry import getRenderer # Validate and fix each document for doc in documents: diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index ef57803d..137dcd05 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.aichat.datamodelFeatureAiChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/aichat/serviceExtraction/__init__.py b/modules/services/serviceExtraction/__init__.py similarity index 100% rename from modules/aichat/serviceExtraction/__init__.py rename to modules/services/serviceExtraction/__init__.py diff --git a/modules/aichat/serviceExtraction/chunking/__init__.py b/modules/services/serviceExtraction/chunking/__init__.py similarity index 100% rename from modules/aichat/serviceExtraction/chunking/__init__.py rename to modules/services/serviceExtraction/chunking/__init__.py diff --git a/modules/aichat/serviceExtraction/chunking/chunkerImage.py b/modules/services/serviceExtraction/chunking/chunkerImage.py similarity index 100% rename from modules/aichat/serviceExtraction/chunking/chunkerImage.py rename to modules/services/serviceExtraction/chunking/chunkerImage.py diff --git a/modules/aichat/serviceExtraction/chunking/chunkerStructure.py b/modules/services/serviceExtraction/chunking/chunkerStructure.py similarity index 100% rename from modules/aichat/serviceExtraction/chunking/chunkerStructure.py rename to modules/services/serviceExtraction/chunking/chunkerStructure.py diff --git a/modules/aichat/serviceExtraction/chunking/chunkerTable.py b/modules/services/serviceExtraction/chunking/chunkerTable.py similarity index 100% rename from modules/aichat/serviceExtraction/chunking/chunkerTable.py rename to modules/services/serviceExtraction/chunking/chunkerTable.py diff --git a/modules/aichat/serviceExtraction/chunking/chunkerText.py b/modules/services/serviceExtraction/chunking/chunkerText.py similarity index 100% rename from modules/aichat/serviceExtraction/chunking/chunkerText.py rename to modules/services/serviceExtraction/chunking/chunkerText.py diff --git a/modules/aichat/serviceExtraction/extractors/__init__.py b/modules/services/serviceExtraction/extractors/__init__.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/__init__.py rename to modules/services/serviceExtraction/extractors/__init__.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorBinary.py b/modules/services/serviceExtraction/extractors/extractorBinary.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorBinary.py rename to modules/services/serviceExtraction/extractors/extractorBinary.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorCsv.py b/modules/services/serviceExtraction/extractors/extractorCsv.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorCsv.py rename to modules/services/serviceExtraction/extractors/extractorCsv.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorDocx.py b/modules/services/serviceExtraction/extractors/extractorDocx.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorDocx.py rename to modules/services/serviceExtraction/extractors/extractorDocx.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorHtml.py b/modules/services/serviceExtraction/extractors/extractorHtml.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorHtml.py rename to modules/services/serviceExtraction/extractors/extractorHtml.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorImage.py b/modules/services/serviceExtraction/extractors/extractorImage.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorImage.py rename to modules/services/serviceExtraction/extractors/extractorImage.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorJson.py b/modules/services/serviceExtraction/extractors/extractorJson.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorJson.py rename to modules/services/serviceExtraction/extractors/extractorJson.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorPdf.py b/modules/services/serviceExtraction/extractors/extractorPdf.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorPdf.py rename to modules/services/serviceExtraction/extractors/extractorPdf.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorPptx.py b/modules/services/serviceExtraction/extractors/extractorPptx.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorPptx.py rename to modules/services/serviceExtraction/extractors/extractorPptx.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorSql.py b/modules/services/serviceExtraction/extractors/extractorSql.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorSql.py rename to modules/services/serviceExtraction/extractors/extractorSql.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorText.py b/modules/services/serviceExtraction/extractors/extractorText.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorText.py rename to modules/services/serviceExtraction/extractors/extractorText.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorXlsx.py b/modules/services/serviceExtraction/extractors/extractorXlsx.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorXlsx.py rename to modules/services/serviceExtraction/extractors/extractorXlsx.py diff --git a/modules/aichat/serviceExtraction/extractors/extractorXml.py b/modules/services/serviceExtraction/extractors/extractorXml.py similarity index 100% rename from modules/aichat/serviceExtraction/extractors/extractorXml.py rename to modules/services/serviceExtraction/extractors/extractorXml.py diff --git a/modules/aichat/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py similarity index 99% rename from modules/aichat/serviceExtraction/mainServiceExtraction.py rename to modules/services/serviceExtraction/mainServiceExtraction.py index 8777dbee..4081158d 100644 --- a/modules/aichat/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -11,10 +11,10 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.aichat.datamodelFeatureAiChat import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall -from modules.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aicore.aicoreModelRegistry import modelRegistry +from modules.aicore.aicoreModelSelector import modelSelector from modules.shared.jsonUtils import stripCodeFences @@ -692,7 +692,6 @@ class ExtractionService: isGenerationResponse = False if options and hasattr(options, 'operationType'): # Generation responses use DATA_GENERATE operation type - from modules.datamodels.datamodelAi import OperationTypeEnum isGenerationResponse = options.operationType == OperationTypeEnum.DATA_GENERATE # Also check if content looks like JSON (starts with { or [) diff --git a/modules/aichat/serviceExtraction/merging/__init__.py b/modules/services/serviceExtraction/merging/__init__.py similarity index 100% rename from modules/aichat/serviceExtraction/merging/__init__.py rename to modules/services/serviceExtraction/merging/__init__.py diff --git a/modules/aichat/serviceExtraction/merging/mergerDefault.py b/modules/services/serviceExtraction/merging/mergerDefault.py similarity index 100% rename from modules/aichat/serviceExtraction/merging/mergerDefault.py rename to modules/services/serviceExtraction/merging/mergerDefault.py diff --git a/modules/aichat/serviceExtraction/merging/mergerTable.py b/modules/services/serviceExtraction/merging/mergerTable.py similarity index 100% rename from modules/aichat/serviceExtraction/merging/mergerTable.py rename to modules/services/serviceExtraction/merging/mergerTable.py diff --git a/modules/aichat/serviceExtraction/merging/mergerText.py b/modules/services/serviceExtraction/merging/mergerText.py similarity index 100% rename from modules/aichat/serviceExtraction/merging/mergerText.py rename to modules/services/serviceExtraction/merging/mergerText.py diff --git a/modules/aichat/serviceExtraction/subMerger.py b/modules/services/serviceExtraction/subMerger.py similarity index 100% rename from modules/aichat/serviceExtraction/subMerger.py rename to modules/services/serviceExtraction/subMerger.py diff --git a/modules/aichat/serviceExtraction/subPipeline.py b/modules/services/serviceExtraction/subPipeline.py similarity index 100% rename from modules/aichat/serviceExtraction/subPipeline.py rename to modules/services/serviceExtraction/subPipeline.py diff --git a/modules/aichat/serviceExtraction/subPromptBuilderExtraction.py b/modules/services/serviceExtraction/subPromptBuilderExtraction.py similarity index 98% rename from modules/aichat/serviceExtraction/subPromptBuilderExtraction.py rename to modules/services/serviceExtraction/subPromptBuilderExtraction.py index 9b785525..8f8f756d 100644 --- a/modules/aichat/serviceExtraction/subPromptBuilderExtraction.py +++ b/modules/services/serviceExtraction/subPromptBuilderExtraction.py @@ -13,7 +13,7 @@ from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, Operati # Type hint for renderer parameter from typing import TYPE_CHECKING if TYPE_CHECKING: - from modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer + from modules.services.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer _RendererLike = BaseRenderer else: _RendererLike = Any diff --git a/modules/aichat/serviceExtraction/subRegistry.py b/modules/services/serviceExtraction/subRegistry.py similarity index 99% rename from modules/aichat/serviceExtraction/subRegistry.py rename to modules/services/serviceExtraction/subRegistry.py index 68c701cc..32727746 100644 --- a/modules/aichat/serviceExtraction/subRegistry.py +++ b/modules/services/serviceExtraction/subRegistry.py @@ -71,7 +71,7 @@ class ExtractorRegistry: module_name = file_path.stem try: # Import the module - module = importlib.import_module(f".{module_name}", package="modules.aichat.serviceExtraction.extractors") + module = importlib.import_module(f".{module_name}", package="modules.services.serviceExtraction.extractors") # Find all extractor classes in the module for attr_name in dir(module): diff --git a/modules/aichat/serviceExtraction/subUtils.py b/modules/services/serviceExtraction/subUtils.py similarity index 100% rename from modules/aichat/serviceExtraction/subUtils.py rename to modules/services/serviceExtraction/subUtils.py diff --git a/modules/aichat/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py similarity index 98% rename from modules/aichat/serviceGeneration/mainServiceGeneration.py rename to modules/services/serviceGeneration/mainServiceGeneration.py index 6953ed7c..a49b78c7 100644 --- a/modules/aichat/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -6,8 +6,8 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.aichat.datamodelFeatureAiChat import ChatDocument -from modules.aichat.serviceGeneration.subDocumentUtility import ( +from modules.datamodels.datamodelChat import ChatDocument +from modules.services.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, detectMimeTypeFromContent, @@ -414,7 +414,7 @@ class GenerationService: continue # Check output style classification (code/document/image/etc.) from renderer - from modules.aichat.serviceGeneration.renderers.registry import getOutputStyle + from modules.services.serviceGeneration.renderers.registry import getOutputStyle outputStyle = getOutputStyle(docFormat) if outputStyle: logger.debug(f"Document {doc.get('id', docIndex)} format '{docFormat}' classified as '{outputStyle}' style") @@ -471,8 +471,8 @@ class GenerationService: Complete document structure with populated elements ready for rendering """ try: - from modules.aichat.serviceGeneration.subStructureGenerator import StructureGenerator - from modules.aichat.serviceGeneration.subContentGenerator import ContentGenerator + from modules.services.serviceGeneration.subStructureGenerator import StructureGenerator + from modules.services.serviceGeneration.subContentGenerator import ContentGenerator # Phase 1: Generate structure skeleton if progressCallback: @@ -537,7 +537,7 @@ class GenerationService: aiService=None ) -> str: """Get adaptive extraction prompt.""" - from modules.aichat.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt + from modules.services.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt return await buildExtractionPrompt( outputFormat=outputFormat, userPrompt=userPrompt, diff --git a/modules/aichat/serviceGeneration/paths/codePath.py b/modules/services/serviceGeneration/paths/codePath.py similarity index 99% rename from modules/aichat/serviceGeneration/paths/codePath.py rename to modules/services/serviceGeneration/paths/codePath.py index 4ab6f79e..f2470385 100644 --- a/modules/aichat/serviceGeneration/paths/codePath.py +++ b/modules/services/serviceGeneration/paths/codePath.py @@ -920,7 +920,7 @@ CRITICAL: def _getCodeRenderer(self, fileType: str): """Get code renderer for file type.""" - from modules.aichat.serviceGeneration.renderers.registry import getRenderer + from modules.services.serviceGeneration.renderers.registry import getRenderer # Map file types to renderer formats formatMap = { diff --git a/modules/aichat/serviceGeneration/paths/documentPath.py b/modules/services/serviceGeneration/paths/documentPath.py similarity index 100% rename from modules/aichat/serviceGeneration/paths/documentPath.py rename to modules/services/serviceGeneration/paths/documentPath.py diff --git a/modules/aichat/serviceGeneration/paths/imagePath.py b/modules/services/serviceGeneration/paths/imagePath.py similarity index 100% rename from modules/aichat/serviceGeneration/paths/imagePath.py rename to modules/services/serviceGeneration/paths/imagePath.py diff --git a/modules/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py b/modules/services/serviceGeneration/renderers/codeRendererBaseTemplate.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/codeRendererBaseTemplate.py rename to modules/services/serviceGeneration/renderers/codeRendererBaseTemplate.py diff --git a/modules/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py b/modules/services/serviceGeneration/renderers/documentRendererBaseTemplate.py similarity index 99% rename from modules/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py rename to modules/services/serviceGeneration/renderers/documentRendererBaseTemplate.py index 76cc1aec..b080ce88 100644 --- a/modules/aichat/serviceGeneration/renderers/documentRendererBaseTemplate.py +++ b/modules/services/serviceGeneration/renderers/documentRendererBaseTemplate.py @@ -81,7 +81,6 @@ class BaseRenderer(ABC): Valid types: "table", "bullet_list", "heading", "paragraph", "code_block", "image" """ # Default: accept all section types - from modules.datamodels.datamodelJson import supportedSectionTypes return list(supportedSectionTypes) @abstractmethod diff --git a/modules/aichat/serviceGeneration/renderers/registry.py b/modules/services/serviceGeneration/renderers/registry.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/registry.py rename to modules/services/serviceGeneration/renderers/registry.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererCodeCsv.py b/modules/services/serviceGeneration/renderers/rendererCodeCsv.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererCodeCsv.py rename to modules/services/serviceGeneration/renderers/rendererCodeCsv.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererCodeJson.py b/modules/services/serviceGeneration/renderers/rendererCodeJson.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererCodeJson.py rename to modules/services/serviceGeneration/renderers/rendererCodeJson.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererCodeXml.py b/modules/services/serviceGeneration/renderers/rendererCodeXml.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererCodeXml.py rename to modules/services/serviceGeneration/renderers/rendererCodeXml.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererCsv.py b/modules/services/serviceGeneration/renderers/rendererCsv.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererCsv.py rename to modules/services/serviceGeneration/renderers/rendererCsv.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererDocx.py b/modules/services/serviceGeneration/renderers/rendererDocx.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererDocx.py rename to modules/services/serviceGeneration/renderers/rendererDocx.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererHtml.py b/modules/services/serviceGeneration/renderers/rendererHtml.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererHtml.py rename to modules/services/serviceGeneration/renderers/rendererHtml.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererImage.py b/modules/services/serviceGeneration/renderers/rendererImage.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererImage.py rename to modules/services/serviceGeneration/renderers/rendererImage.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererJson.py b/modules/services/serviceGeneration/renderers/rendererJson.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererJson.py rename to modules/services/serviceGeneration/renderers/rendererJson.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererMarkdown.py b/modules/services/serviceGeneration/renderers/rendererMarkdown.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererMarkdown.py rename to modules/services/serviceGeneration/renderers/rendererMarkdown.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererPdf.py b/modules/services/serviceGeneration/renderers/rendererPdf.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererPdf.py rename to modules/services/serviceGeneration/renderers/rendererPdf.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererPptx.py b/modules/services/serviceGeneration/renderers/rendererPptx.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererPptx.py rename to modules/services/serviceGeneration/renderers/rendererPptx.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererText.py b/modules/services/serviceGeneration/renderers/rendererText.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererText.py rename to modules/services/serviceGeneration/renderers/rendererText.py diff --git a/modules/aichat/serviceGeneration/renderers/rendererXlsx.py b/modules/services/serviceGeneration/renderers/rendererXlsx.py similarity index 100% rename from modules/aichat/serviceGeneration/renderers/rendererXlsx.py rename to modules/services/serviceGeneration/renderers/rendererXlsx.py diff --git a/modules/aichat/serviceGeneration/subContentGenerator.py b/modules/services/serviceGeneration/subContentGenerator.py similarity index 99% rename from modules/aichat/serviceGeneration/subContentGenerator.py rename to modules/services/serviceGeneration/subContentGenerator.py index e5c9eec1..86464ef6 100644 --- a/modules/aichat/serviceGeneration/subContentGenerator.py +++ b/modules/services/serviceGeneration/subContentGenerator.py @@ -12,7 +12,7 @@ import base64 import re import traceback from typing import Dict, Any, Optional, List, Callable -from modules.aichat.serviceGeneration.subContentIntegrator import ContentIntegrator +from modules.services.serviceGeneration.subContentIntegrator import ContentIntegrator from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/aichat/serviceGeneration/subContentIntegrator.py b/modules/services/serviceGeneration/subContentIntegrator.py similarity index 100% rename from modules/aichat/serviceGeneration/subContentIntegrator.py rename to modules/services/serviceGeneration/subContentIntegrator.py diff --git a/modules/aichat/serviceGeneration/subDocumentUtility.py b/modules/services/serviceGeneration/subDocumentUtility.py similarity index 100% rename from modules/aichat/serviceGeneration/subDocumentUtility.py rename to modules/services/serviceGeneration/subDocumentUtility.py diff --git a/modules/aichat/serviceGeneration/subJsonSchema.py b/modules/services/serviceGeneration/subJsonSchema.py similarity index 100% rename from modules/aichat/serviceGeneration/subJsonSchema.py rename to modules/services/serviceGeneration/subJsonSchema.py diff --git a/modules/aichat/serviceGeneration/subPromptBuilderGeneration.py b/modules/services/serviceGeneration/subPromptBuilderGeneration.py similarity index 100% rename from modules/aichat/serviceGeneration/subPromptBuilderGeneration.py rename to modules/services/serviceGeneration/subPromptBuilderGeneration.py diff --git a/modules/aichat/serviceGeneration/subStructureGenerator.py b/modules/services/serviceGeneration/subStructureGenerator.py similarity index 100% rename from modules/aichat/serviceGeneration/subStructureGenerator.py rename to modules/services/serviceGeneration/subStructureGenerator.py diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 83cb9327..2c975f1d 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -161,7 +161,7 @@ class UtilsService: Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChat. """ try: - from modules.aichat.interfaceFeatureAiChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/aichat/serviceWeb/mainServiceWeb.py b/modules/services/serviceWeb/mainServiceWeb.py similarity index 100% rename from modules/aichat/serviceWeb/mainServiceWeb.py rename to modules/services/serviceWeb/mainServiceWeb.py diff --git a/modules/workflows/automation/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py index 332f8d29..23bbf125 100644 --- a/modules/workflows/automation/mainWorkflow.py +++ b/modules/workflows/automation/mainWorkflow.py @@ -12,7 +12,8 @@ import logging import json from typing import Dict, Any, Optional -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index d6264bed..39d6e16f 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index ebdada6a..4f9bbd21 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 4451f0ae..65e95a32 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index 298ecbe4..cdaf55b6 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart @@ -134,7 +134,6 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult: logger.warning(f"Error extracting context from documents in simple mode: {e}") # Use direct AI call without document generation pipeline - from modules.datamodels.datamodelAi import AiCallRequest, OperationTypeEnum, ProcessingModeEnum request = AiCallRequest( prompt=aiPrompt, context=context_text if context_text else None, @@ -170,7 +169,6 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult: isImageGeneration = normalized_result_type in imageFormats if normalized_result_type else False # Build options with correct operationType - from modules.datamodels.datamodelAi import OperationTypeEnum # resultFormat in options can be None - formats will be determined by AI if not provided options = AiCallOptions( resultFormat=output_format, # Can be None - formats determined by AI diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index 4dc7c4ef..e32c1965 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index 07350a54..bb6f8437 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 753319fe..62b43bce 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index 75bf8771..ff7e896f 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index 5fd926a1..5b90ce13 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index 6b4dabe9..9991285b 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index c871fa06..8e3b7185 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index e299d226..2f011a25 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index bd4f666c..45b60cad 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index 3b07ca6a..cbec7960 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index b172140b..631795b3 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index fd1570f3..55d99654 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index 874867b8..b997889e 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 8333499f..2bd7ab74 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index 650de4ce..bbdc2cc7 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index 9fb0cc3f..5ac4e548 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 69beed2d..59604896 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index c64abf18..2d325d9f 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index 4bb1861d..f8831d59 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index 7dff3f3f..9b7fb011 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index e0e2b811..a4bf18b6 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index 418ba477..f149e482 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index 9cec8d61..c64a6637 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 7942ab21..722dbc99 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index 063fd3c5..62b6dd94 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index a9b766c1..318271c3 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index 13c31219..73cdb730 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index 469dde0d..e9361853 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index bd0618d7..1f469b80 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 2d657596..0e4d6ee4 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.aichat.datamodelFeatureAiChat import ActionResult, ActionItem, TaskStep -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index af9c3f08..a4ae05e9 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.aichat.datamodelFeatureAiChat import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 62f9a72a..94c695cb 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,6 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.aichat.datamodelFeatureAiChat import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index 084024aa..e3131939 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.aichat.datamodelFeatureAiChat import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index f4533e7d..770c868a 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskResult, ActionItem -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index 834353b7..45b92961 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.aichat.datamodelFeatureAiChat import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,6 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.aichat.datamodelFeatureAiChat import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1041,6 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.aichat.datamodelFeatureAiChat import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 405aabe1..1cdf0d53 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.aichat.datamodelFeatureAiChat import TaskStep, ActionResult, Observation +from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index b70d632c..136dd2cb 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.aichat.datamodelFeatureAiChat import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.aichat.datamodelFeatureAiChat import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict @@ -452,7 +452,7 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.aichat.interfaceFeatureAiChat as interfaceDbChat + import modules.interfaces.interfaceDbChat as interfaceDbChat from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() interfaceDbChat = interfaceDbChat.getInterface(rootInterface.currentUser) diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index e8db412d..31878033 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index 08007833..11a54ca1 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.aichat.datamodelFeatureAiChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index c8f6b09c..c73d280b 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -7,8 +7,8 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING from modules.datamodels import datamodelChat -from modules.aichat.datamodelFeatureAiChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -468,7 +468,6 @@ class WorkflowProcessor: ) # Prepare AI call options for fast path (balanced, fast processing) - from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest options = AiCallOptions( operationType=OperationTypeEnum.DATA_ANALYSE, @@ -494,7 +493,6 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.aichat.datamodelFeatureAiChat import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -538,8 +536,6 @@ class WorkflowProcessor: UnderstandingResult with all understanding components """ try: - from modules.datamodels.datamodelWorkflow import UnderstandingResult, TaskDefinition - from modules.shared.jsonUtils import parseJsonWithModel # Ensure AI service is initialized await self.services.ai.ensureAiObjectsInitialized() @@ -600,7 +596,6 @@ class WorkflowProcessor: except Exception as e: logger.error(f"Error in initialUnderstanding: {str(e)}") # Return minimal UnderstandingResult on error - from modules.datamodels.datamodelWorkflow import UnderstandingResult return UnderstandingResult( parameters={"language": context.userLanguage}, intention={"primaryGoal": context.originalPrompt}, @@ -626,8 +621,6 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.aichat.datamodelFeatureAiChat import ChatMessage, ChatDocument, ActionDocument - from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status checkWorkflowStopped(self.services) diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index 417bb384..d0c35bf1 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.aichat.datamodelFeatureAiChat import ( +from modules.datamodels.datamodelChat import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.aichat.datamodelFeatureAiChat import TaskContext +from modules.datamodels.datamodelChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,6 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +791,6 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.aichat.datamodelFeatureAiChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +919,6 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.aichat.datamodelFeatureAiChat import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/scripts/.$import_diagram.drawio.bkp b/scripts/.$import_diagram.drawio.bkp new file mode 100644 index 00000000..588285b0 --- /dev/null +++ b/scripts/.$import_diagram.drawio.bkp @@ -0,0 +1,2723 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/.$import_diagram_containers.drawio.bkp b/scripts/.$import_diagram_containers.drawio.bkp new file mode 100644 index 00000000..cad95eef --- /dev/null +++ b/scripts/.$import_diagram_containers.drawio.bkp @@ -0,0 +1,317 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/function_imports_analysis.txt b/scripts/function_imports_analysis.txt new file mode 100644 index 00000000..23f26c9f --- /dev/null +++ b/scripts/function_imports_analysis.txt @@ -0,0 +1,435 @@ +================================================================================ +FUNCTION IMPORTS ANALYSIS +================================================================================ + +Total function imports (internal modules): 226 + - CIRCULAR (must stay): 4 + - REDUNDANT (can remove): 0 + - MOVABLE (can move): 222 + + +================================================================================ +MOVABLE TO HEADER (grouped by source module) +These imports could potentially be moved to the module header. +================================================================================ + +gateway.app +----------- + [lifespan] modules.shared.auditLogger + +gateway.modules.aichat.datamodelFeatureAiChat +--------------------------------------------- + [updateFromSelection] modules.datamodels.datamodelWorkflow + +gateway.modules.aichat.interfaceFeatureAiChat +--------------------------------------------- + [_enrichAutomationsWithUserAndMandate] modules.interfaces.interfaceDbApp + [storeDebugMessageAndDocuments] modules.interfaces.interfaceDbManagement + [setUserContext] modules.security.rootAccess + [_notifyAutomationChanged] modules.shared.callbackRegistry + [storeDebugMessageAndDocuments] modules.shared.debugLogger + +gateway.modules.aichat.serviceAi.mainServiceAi +---------------------------------------------- + [renderResult] modules.aichat.serviceGeneration.mainServiceGeneration + [_handleCodeGeneration] modules.aichat.serviceGeneration.paths.codePath + [_handleDocumentGeneration] modules.aichat.serviceGeneration.paths.documentPath + [_handleImageGeneration] modules.aichat.serviceGeneration.paths.imagePath + +gateway.modules.aichat.serviceAi.subContentExtraction +----------------------------------------------------- + [extractTextFromImage] modules.datamodels.datamodelAi + [processTextContentWithAi] modules.datamodels.datamodelAi + +gateway.modules.aichat.serviceAi.subJsonResponseHandling +-------------------------------------------------------- + [mergeFragmentIntoSection] modules.shared.debugLogger + +gateway.modules.aichat.serviceAi.subStructureFilling +---------------------------------------------------- + [_getAcceptedSectionTypesForFormat] modules.aichat.serviceGeneration.renderers.registry + [_getAcceptedSectionTypesForFormat] modules.datamodels.datamodelJson + [buildSectionPromptWithContinuation] modules.shared.jsonContinuation + [_extractAndMergeMultipleJsonBlocks] modules.shared.jsonUtils + [_processAiResponseForSection] modules.shared.jsonUtils + [_processSingleSection] modules.shared.jsonUtils + +gateway.modules.aichat.serviceAi.subStructureGeneration +------------------------------------------------------- + [generateStructure] modules.aichat.serviceGeneration.renderers.registry + [generateStructure] modules.shared + [generateStructure] modules.shared.jsonContinuation + +gateway.modules.aichat.serviceExtraction.mainServiceExtraction +-------------------------------------------------------------- + [extractContent] modules.interfaces.interfaceDbManagement + [extractContent] modules.shared.debugLogger + +gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction +------------------------------------------------------------------- + [buildExtractionPrompt] modules.shared.debugLogger + +gateway.modules.aichat.serviceGeneration.mainServiceGeneration +-------------------------------------------------------------- + [getAdaptiveExtractionPrompt] modules.aichat.serviceExtraction.subPromptBuilderExtraction + [renderReport] modules.aichat.serviceGeneration.renderers.registry + [generateDocumentWithTwoPhases] modules.aichat.serviceGeneration.subContentGenerator + [generateDocumentWithTwoPhases] modules.aichat.serviceGeneration.subStructureGenerator + +gateway.modules.aichat.serviceGeneration.paths.codePath +------------------------------------------------------- + [_getCodeRenderer] modules.aichat.serviceGeneration.renderers.registry + [generateCode] modules.datamodels.datamodelDocument + [_generateCodeStructure] modules.shared.jsonContinuation + [_generateSingleFileContent] modules.shared.jsonContinuation + +gateway.modules.aichat.serviceGeneration.renderers.rendererDocx +--------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererHtml +--------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererImage +---------------------------------------------------------------- + [_compressPromptWithAi] modules.datamodels.datamodelAi + [_generateAiImage] modules.datamodels.datamodelAi + +gateway.modules.aichat.serviceGeneration.renderers.rendererJson +--------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown +------------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererPdf +-------------------------------------------------------------- + [_getAiStylesWithPdfColors] modules.datamodels.datamodelAi + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererPptx +--------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererText +--------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx +--------------------------------------------------------------- + [_getAiStylesWithExcelColors] modules.datamodels.datamodelAi + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.aichat.serviceGeneration.subContentGenerator +------------------------------------------------------------ + [_generateImageSection] modules.datamodels.datamodelAi + [_generateSimpleSection] modules.datamodels.datamodelAi + [_generateSimpleSection] modules.shared.jsonUtils + +gateway.modules.aichat.serviceGeneration.subStructureGenerator +-------------------------------------------------------------- + [generateStructure] modules.datamodels.datamodelAi + +gateway.modules.auth.authentication +----------------------------------- + [requireSysAdmin] modules.shared.auditLogger + +gateway.modules.auth.tokenManager +--------------------------------- + [getFreshToken] modules.interfaces.interfaceDbApp + [getFreshToken] modules.security.rootAccess + +gateway.modules.auth.tokenRefreshService +---------------------------------------- + [_refresh_google_token] modules.auth.tokenManager + [_refresh_microsoft_token] modules.auth.tokenManager + [proactive_refresh] modules.interfaces.interfaceDbApp + [refresh_expired_tokens] modules.interfaces.interfaceDbApp + [proactive_refresh] modules.security.rootAccess + [refresh_expired_tokens] modules.security.rootAccess + +gateway.modules.features.automation.routeFeatureAutomation +---------------------------------------------------------- + [execute_automation] modules.services + +gateway.modules.features.chatbot.datamodelFeatureChatbot +-------------------------------------------------------- + [updateFromSelection] modules.datamodels.datamodelWorkflow + +gateway.modules.features.chatbot.interfaceFeatureChatbot +-------------------------------------------------------- + [createLog] modules.features.chatbot.eventManager + [createMessage] modules.features.chatbot.eventManager + [_enrichAutomationsWithUserAndMandate] modules.interfaces.interfaceDbApp + [storeDebugMessageAndDocuments] modules.interfaces.interfaceDbManagement + [setUserContext] modules.security.rootAccess + [_notifyAutomationChanged] modules.shared.callbackRegistry + [storeDebugMessageAndDocuments] modules.shared.debugLogger + [deleteAutomationDefinition] modules.shared.eventManagement + +gateway.modules.features.chatbot.mainChatbot +-------------------------------------------- + [_convert_file_ids_to_document_references] modules.interfaces.interfaceRbac + +gateway.modules.features.neutralizer.mainNeutralizePlayground +------------------------------------------------------------- + [processSharepointFiles] modules.services.serviceSharepoint.mainServiceSharepoint + +gateway.modules.features.realestate.interfaceFeatureRealEstate +-------------------------------------------------------------- + [setUserContext] modules.security.rootAccess + +gateway.modules.features.realestate.mainRealEstate +-------------------------------------------------- + [executeIntentBasedOperation] modules.features.realestate.datamodelFeatureRealEstate + +gateway.modules.features.trustee.interfaceFeatureTrustee +-------------------------------------------------------- + [setUserContext] modules.security.rootAccess + +gateway.modules.interfaces.interfaceBootstrap +--------------------------------------------- + [_applyDatabaseOptimizations] modules.shared.dbMultiTenantOptimizations + +gateway.modules.interfaces.interfaceDbApp +----------------------------------------- + [getRootInterface] modules.security.rootAccess + +gateway.modules.interfaces.interfaceDbManagement +------------------------------------------------ + [_initializeStandardPrompts] modules.interfaces.interfaceDbApp + [_initializeStandardPrompts] modules.security.rootAccess + [setUserContext] modules.security.rootAccess + +gateway.modules.interfaces.interfaceFeatures +-------------------------------------------- + [syncRolesFromTemplate] modules.datamodels.datamodelMembership + +gateway.modules.interfaces.interfaceRbac +---------------------------------------- + [getRecordsetWithRBAC] modules.connectors.connectorDbPostgre + +gateway.modules.interfaces.interfaceTicketObjects +------------------------------------------------- + [createTicketInterfaceByType] modules.connectors.connectorTicketsClickup + [createTicketInterfaceByType] modules.connectors.connectorTicketsJira + +gateway.modules.routes.routeAdminAutomationEvents +------------------------------------------------- + [sync_all_automation_events] modules.interfaces.interfaceDbApp + [sync_all_automation_events] modules.services + [get_all_automation_events] modules.shared.eventManagement + [remove_event] modules.shared.eventManagement + [sync_all_automation_events] modules.workflows.automation + +gateway.modules.routes.routeAdminFeatures +----------------------------------------- + [_getInstancePermissions] modules.datamodels.datamodelMembership + [_getUserRoleInInstance] modules.datamodels.datamodelMembership + [addUserToFeatureInstance] modules.datamodels.datamodelMembership + [listFeatureInstanceUsers] modules.datamodels.datamodelMembership + [removeUserFromFeatureInstance] modules.datamodels.datamodelMembership + [updateFeatureInstanceUserRoles] modules.datamodels.datamodelMembership + [_getInstancePermissions] modules.datamodels.datamodelRbac + [_getUserRoleInInstance] modules.datamodels.datamodelRbac + [_hasMandateAdminRole] modules.datamodels.datamodelRbac + [getFeatureInstanceAvailableRoles] modules.datamodels.datamodelRbac + [listFeatureInstanceUsers] modules.datamodels.datamodelRbac + +gateway.modules.routes.routeDataUsers +------------------------------------- + [delete_user] modules.datamodels.datamodelMembership + [get_user] modules.datamodels.datamodelMembership + [reset_user_password] modules.datamodels.datamodelMembership + [sendPasswordLink] modules.datamodels.datamodelMembership + [update_user] modules.datamodels.datamodelMembership + [sendPasswordLink] modules.services + [change_password] modules.shared.auditLogger + [reset_user_password] modules.shared.auditLogger + [sendPasswordLink] modules.shared.auditLogger + [sendPasswordLink] modules.shared.configuration + +gateway.modules.routes.routeDataWorkflows +----------------------------------------- + [get_action_schema] modules.services + [get_all_actions] modules.services + [get_method_actions] modules.services + [get_action_schema] modules.workflows.processing.shared.methodDiscovery + [get_all_actions] modules.workflows.processing.shared.methodDiscovery + [get_method_actions] modules.workflows.processing.shared.methodDiscovery + +gateway.modules.routes.routeGdpr +-------------------------------- + [exportUserData] modules.datamodels.datamodelFeatures + [deleteAccount] modules.datamodels.datamodelInvitation + [exportUserData] modules.datamodels.datamodelInvitation + [deleteAccount] modules.datamodels.datamodelMembership + [exportPortableData] modules.datamodels.datamodelMembership + [exportUserData] modules.datamodels.datamodelMembership + [deleteAccount] modules.datamodels.datamodelSecurity + +gateway.modules.routes.routeInvitations +--------------------------------------- + [createInvitation] modules.datamodels.datamodelFeatures + [_hasMandateAdminRole] modules.datamodels.datamodelRbac + [_isInstanceRole] modules.datamodels.datamodelRbac + [createInvitation] modules.datamodels.datamodelRbac + [registerAndAcceptInvitation] modules.security.passwordUtils + [createInvitation] modules.shared.configuration + [listInvitations] modules.shared.configuration + +gateway.modules.routes.routeMessaging +------------------------------------- + [_hasTriggerPermission] modules.interfaces.interfaceDbApp + [triggerSubscription] modules.services + +gateway.modules.routes.routeSecurityAdmin +----------------------------------------- + [revoke_tokens_by_mandate] modules.datamodels.datamodelMembership + +gateway.modules.routes.routeSecurityGoogle +------------------------------------------ + [auth_callback] modules.datamodels.datamodelSecurity + [logout] modules.shared.auditLogger + +gateway.modules.routes.routeSecurityLocal +----------------------------------------- + [_sendAuthEmail] modules.datamodels.datamodelMessaging + [_sendAuthEmail] modules.interfaces.interfaceMessaging + [login] modules.shared.auditLogger + [logout] modules.shared.auditLogger + [passwordReset] modules.shared.auditLogger + +gateway.modules.routes.routeSecurityMsft +---------------------------------------- + [logout] modules.shared.auditLogger + +gateway.modules.security.rootAccess +----------------------------------- + [_ensureBootstrap] modules.interfaces.interfaceBootstrap + +gateway.modules.services.__init__ +--------------------------------- + [__init__] modules.interfaces.interfaceDbApp + [__init__] modules.interfaces.interfaceDbManagement + +gateway.modules.services.serviceChat.mainServiceChat +---------------------------------------------------- + [getChatDocumentsFromDocumentList] modules.datamodels.datamodelDocref + +gateway.modules.services.serviceUtils.mainServiceUtils +------------------------------------------------------ + [storeDebugMessageAndDocuments] modules.aichat.interfaceFeatureAiChat + [debugLogToFile] modules.shared.debugLogger + [writeDebugArtifact] modules.shared.debugLogger + [writeDebugFile] modules.shared.debugLogger + +gateway.modules.shared.auditLogger +---------------------------------- + [_ensureInitialized] modules.datamodels.datamodelAudit + [cleanupOldEntries] modules.datamodels.datamodelAudit + [getAuditLogs] modules.datamodels.datamodelAudit + [logEvent] modules.datamodels.datamodelAudit + [registerAuditLogCleanupScheduler] modules.shared.eventManagement + +gateway.modules.shared.debugLogger +---------------------------------- + [debugLogToFile] modules.shared.timeUtils + +gateway.modules.shared.jsonUtils +-------------------------------- + [buildContinuationContext] modules.shared.jsonContinuation + +gateway.modules.workflows.automation.subAutomationSchedule +---------------------------------------------------------- + [start] modules.shared.callbackRegistry + [start] modules.workflows.automation + +gateway.modules.workflows.methods.methodAi.actions.generateCode +--------------------------------------------------------------- + [generateCode] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodAi.actions.generateDocument +------------------------------------------------------------------- + [generateDocument] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodAi.actions.process +---------------------------------------------------------- + [process] modules.datamodels.datamodelDocref + [process] modules.datamodels.datamodelWorkflow + +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase +--------------------------------------------------------------------- + [queryDatabase] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext +--------------------------------------------------------------------------------------- + [composeAndDraftEmailWithContext] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail +---------------------------------------------------------------------- + [sendDraftEmail] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile +------------------------------------------------------------------- + [copyFile] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath +----------------------------------------------------------------------------- + [downloadFileByPath] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile +--------------------------------------------------------------------- + [uploadFile] modules.datamodels.datamodelDocref + +gateway.modules.workflows.methods.methodSharepoint.helpers.documentParsing +-------------------------------------------------------------------------- + [parseDocumentListForFolder] modules.datamodels.datamodelDocref + [parseDocumentListForFoundDocuments] modules.datamodels.datamodelDocref + +gateway.modules.workflows.processing.core.actionExecutor +-------------------------------------------------------- + [_createActionCompletionMessage] modules.workflows.processing.core.messageCreator + +gateway.modules.workflows.processing.modes.modeDynamic +------------------------------------------------------ + [_actExecute] modules.datamodels.datamodelAi + [_planSelect] modules.datamodels.datamodelAi + [_refineDecide] modules.datamodels.datamodelAi + [_actExecute] modules.datamodels.datamodelDocref + [_planSelect] modules.datamodels.datamodelDocref + [_actExecute] modules.datamodels.datamodelWorkflow + [_planSelect] modules.datamodels.datamodelWorkflow + [_actExecute] modules.shared.jsonUtils + [_planSelect] modules.shared.jsonUtils + [_refineDecide] modules.shared.jsonUtils + [_actExecute] modules.workflows.processing.shared.methodDiscovery + +gateway.modules.workflows.processing.shared.placeholderFactory +-------------------------------------------------------------- + [extractReviewContent] modules.aichat.datamodelFeatureAiChat + [extractLatestRefinementFeedback] modules.aichat.interfaceFeatureAiChat + [extractLatestRefinementFeedback] modules.interfaces.interfaceDbApp + +gateway.modules.workflows.workflowManager +----------------------------------------- + [_executeTasks] modules.datamodels.datamodelWorkflow + [workflowStart] modules.workflows.processing.shared.methodDiscovery + [_checkIfHistoryAvailable] modules.workflows.processing.shared.placeholderFactory + + +================================================================================ +CIRCULAR DEPENDENCY (must stay in function) +================================================================================ + +gateway.modules.shared.auditLogger +---------------------------------- + [_ensureInitialized] modules.connectors.connectorDbPostgre + +gateway.modules.shared.configuration +------------------------------------ + [decryptValue] modules.shared.auditLogger + [encryptValue] modules.shared.auditLogger + [get] modules.shared.auditLogger \ No newline at end of file diff --git a/scripts/import_analysis.csv b/scripts/import_analysis.csv index 00533906..03015969 100644 --- a/scripts/import_analysis.csv +++ b/scripts/import_analysis.csv @@ -18,6 +18,7 @@ gateway.app,modules.routes.routeAdminFeatures,header,Yes gateway.app,modules.routes.routeAdminRbacExport,header,Yes gateway.app,modules.routes.routeAdminRbacRules,header,Yes gateway.app,modules.routes.routeAttributes,header,Yes +gateway.app,modules.routes.routeChat,header,Yes gateway.app,modules.routes.routeDataConnections,header,Yes gateway.app,modules.routes.routeDataFiles,header,Yes gateway.app,modules.routes.routeDataMandates,header,Yes @@ -41,680 +42,75 @@ gateway.app,os,header,Yes gateway.app,sys,header,Yes gateway.app,unicodedata,header,Yes gateway.app,urllib.parse,header,Yes -gateway.modules.aichat.aicore.aicoreBase,abc,header,Yes -gateway.modules.aichat.aicore.aicoreBase,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicoreBase,time,function getCachedModels,Yes -gateway.modules.aichat.aicore.aicoreBase,typing,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,importlib,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,logging,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,modules.connectors.connectorDbPostgre,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,modules.datamodels.datamodelUam,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,modules.security.rbac,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,modules.security.rbacHelpers,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,os,header,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,time,function refreshModels,Yes -gateway.modules.aichat.aicore.aicoreModelRegistry,typing,header,Yes -gateway.modules.aichat.aicore.aicoreModelSelector,logging,header,Yes -gateway.modules.aichat.aicore.aicoreModelSelector,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicoreModelSelector,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,base64,function callAiImage,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,fastapi,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,httpx,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,logging,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,modules.shared.configuration,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,os,header,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,time,function callAiImage,Yes -gateway.modules.aichat.aicore.aicorePluginAnthropic,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginInternal,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicorePluginInternal,logging,header,Yes -gateway.modules.aichat.aicore.aicorePluginInternal,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicorePluginInternal,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,fastapi,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,httpx,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,json,function generateImage,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,logging,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,function generateImage,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,modules.shared.configuration,header,Yes -gateway.modules.aichat.aicore.aicorePluginOpenai,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,fastapi,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,httpx,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webSearch,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,json,function webCrawl,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,logging,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,function _testConnection,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.datamodels.datamodelTools,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,modules.shared.configuration,header,Yes -gateway.modules.aichat.aicore.aicorePluginPerplexity,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,(relative) .aicoreBase,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,asyncio,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,dataclasses,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webSearch,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,json,function webCrawl,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,logging,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,modules.datamodels.datamodelTools,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,modules.shared.configuration,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,re,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,re,function _cleanUrl,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,tavily,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,typing,header,Yes -gateway.modules.aichat.aicore.aicorePluginTavily,urllib.parse,function _normalizeUrl,Yes -gateway.modules.aichat.datamodelFeatureAiChat,enum,header,Yes -gateway.modules.aichat.datamodelFeatureAiChat,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes -gateway.modules.aichat.datamodelFeatureAiChat,modules.shared.attributeUtils,header,Yes -gateway.modules.aichat.datamodelFeatureAiChat,modules.shared.timeUtils,header,Yes -gateway.modules.aichat.datamodelFeatureAiChat,pydantic,header,Yes -gateway.modules.aichat.datamodelFeatureAiChat,typing,header,Yes -gateway.modules.aichat.datamodelFeatureAiChat,uuid,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,asyncio,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,datetime,function storeDebugMessageAndDocuments,Yes -gateway.modules.aichat.interfaceFeatureAiChat,json,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,logging,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,math,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.connectors.connectorDbPostgre,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelPagination,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelRbac,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.datamodels.datamodelUam,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.interfaces.interfaceRbac,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.security.rbac,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.security.rootAccess,function setUserContext,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.configuration,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes -gateway.modules.aichat.interfaceFeatureAiChat,modules.shared.timeUtils,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,os,function storeDebugMessageAndDocuments,Yes -gateway.modules.aichat.interfaceFeatureAiChat,typing,header,Yes -gateway.modules.aichat.interfaceFeatureAiChat,uuid,header,Yes -gateway.modules.aichat.mainAiChat,(relative) .aicore.aicoreModelRegistry,function onStart,Yes -gateway.modules.aichat.mainAiChat,logging,header,Yes -gateway.modules.aichat.mainAiChat,typing,header,Yes -gateway.modules.aichat.routeFeatureAiChat,(relative) .,header,Yes -gateway.modules.aichat.routeFeatureAiChat,(relative) .datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.routeFeatureAiChat,fastapi,header,Yes -gateway.modules.aichat.routeFeatureAiChat,logging,header,Yes -gateway.modules.aichat.routeFeatureAiChat,modules.auth,header,Yes -gateway.modules.aichat.routeFeatureAiChat,modules.workflows.automation,header,Yes -gateway.modules.aichat.routeFeatureAiChat,typing,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subAiCallLooping,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subContentExtraction,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subDocumentIntents,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subResponseParsing,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subStructureFilling,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,(relative) .subStructureGeneration,function _initializeSubmodules,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,base64,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,json,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,logging,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceExtraction.mainServiceExtraction,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.mainServiceGeneration,function renderResult,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.codePath,function _handleCodeGeneration,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.documentPath,function _handleDocumentGeneration,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.aichat.serviceGeneration.paths.imagePath,function _handleImageGeneration,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.interfaces.interfaceAiObjects,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,re,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,time,header,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,time,function _handleDataExtraction,Yes -gateway.modules.aichat.serviceAi.mainServiceAi,typing,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,(relative) .subLoopingUseCases,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,json,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,logging,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonContinuation,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceAi.subAiCallLooping,typing,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,base64,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,json,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,logging,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function extractTextFromImage,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function processTextContentWithAi,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,function extractAndPrepareContent,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,traceback,function extractTextFromImage,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,traceback,function processTextContentWithAi,Yes -gateway.modules.aichat.serviceAi.subContentExtraction,typing,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,json,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,logging,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,function resolvePreExtractedDocument,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,traceback,function resolvePreExtractedDocument,Yes -gateway.modules.aichat.serviceAi.subDocumentIntents,typing,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,datetime,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,json,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,logging,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,os,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,re,header,Yes -gateway.modules.aichat.serviceAi.subJsonMerger,typing,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,(relative) .subJsonMerger,function mergeJsonStringsWithOverlap,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,json,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,logging,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.debugLogger,function mergeFragmentIntoSection,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function mergeJsonStringsWithOverlap,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _normalizeToElementsStructure,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractRowsFromFragment,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractOverlapAndContinuation,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeWithExplicitOverlap,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _smartConcatenate,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStringsWithOverlapFallback,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _detectAndNormalizeFragment,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _extractValidJsonPrefix,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function _mergeJsonStructuresGeneric,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,function extractKpiValuesFromIncompleteJson,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,header,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,function _extractRowsFromFragment,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,re,function _detectAndNormalizeFragment,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,traceback,function _mergeJsonStructuresGeneric,Yes -gateway.modules.aichat.serviceAi.subJsonResponseHandling,typing,header,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,dataclasses,header,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleChapterStructureFinalResult,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeStructureFinalResult,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,json,function _handleCodeContentFinalResult,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,logging,header,Yes -gateway.modules.aichat.serviceAi.subLoopingUseCases,typing,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,(relative) .subJsonResponseHandling,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,json,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,logging,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceAi.subResponseParsing,typing,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,asyncio,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,base64,function _processAiResponseForSection,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,copy,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,json,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,logging,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.aichat.serviceGeneration.renderers.registry,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonContinuation,function buildSectionPromptWithContinuation,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _extractAndMergeMultipleJsonBlocks,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processAiResponseForSection,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processSingleSection,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceAi.subStructureFilling,typing,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,json,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,logging,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.aichat.serviceGeneration.renderers.registry,function generateStructure,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.shared,function generateStructure,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.shared.jsonContinuation,function generateStructure,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceAi.subStructureGeneration,typing,header,Yes -gateway.modules.aichat.serviceExtraction.__init__,(relative) .mainServiceExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,PIL,function chunk,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,base64,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,io,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerImage,typing,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,json,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerStructure,typing,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerTable,typing,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerText,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerText,logging,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.chunking.chunkerText,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,base64,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorBinary,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorCsv,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,docx,function _load,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,io,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorDocx,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,bs4,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorHtml,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,PIL,function extract,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,base64,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,io,function extract,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,logging,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorImage,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorJson,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorJson,json,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorJson,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorJson,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,PyPDF2,function _load,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,base64,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,fitz,function _load,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,io,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPdf,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,base64,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,io,function extract,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,logging,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,pptx,function _load,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorPptx,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorSql,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorSql,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorSql,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorText,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorText,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorText,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,datetime,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,io,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,openpyxl,function _load,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXlsx,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXml,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXml,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXml,typing,header,Yes -gateway.modules.aichat.serviceExtraction.extractors.extractorXml,xml.etree.ElementTree,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerDefault,function applyMerging,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerTable,function applyMerging,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerText,function applyMerging,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subMerger,function applyMerging,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subPipeline,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,(relative) .subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,asyncio,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,base64,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,json,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,logging,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.aicore.aicoreModelSelector,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,function mergePartResults,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.interfaces.interfaceDbManagement,function extractContent,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.shared.debugLogger,function extractContent,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,time,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,typing,header,Yes -gateway.modules.aichat.serviceExtraction.mainServiceExtraction,uuid,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerDefault,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerDefault,typing,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerTable,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerTable,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerTable,typing,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerText,(relative) ..subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerText,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.merging.mergerText,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subMerger,(relative) .subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.subMerger,logging,header,Yes -gateway.modules.aichat.serviceExtraction.subMerger,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.subMerger,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .mainServiceExtraction,function runExtraction,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .subRegistry,header,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,(relative) .subUtils,header,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,logging,header,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.subPipeline,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,json,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,logging,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,modules.shared.debugLogger,function buildExtractionPrompt,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerImage,function __init__,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerStructure,function __init__,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerTable,function __init__,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .chunking.chunkerText,function __init__,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,(relative) .extractors.extractorBinary,function _auto_discover_extractors,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,importlib,function _auto_discover_extractors,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,logging,header,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,os,function _auto_discover_extractors,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,pathlib,function _auto_discover_extractors,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,traceback,function _auto_discover_extractors,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,traceback,function __init__,Yes -gateway.modules.aichat.serviceExtraction.subRegistry,typing,header,Yes -gateway.modules.aichat.serviceExtraction.subUtils,uuid,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,(relative) .renderers.registry,function _getFormatRenderer,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,base64,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,logging,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceExtraction.subPromptBuilderExtraction,function getAdaptiveExtractionPrompt,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.renderers.registry,function renderReport,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subContentGenerator,function generateDocumentWithTwoPhases,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subDocumentUtility,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.aichat.serviceGeneration.subStructureGenerator,function generateDocumentWithTwoPhases,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,traceback,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,typing,header,Yes -gateway.modules.aichat.serviceGeneration.mainServiceGeneration,uuid,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,json,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,logging,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.aichat.serviceGeneration.renderers.registry,function _getCodeRenderer,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelDocument,function generateCode,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateCodeStructure,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateSingleFileContent,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,modules.shared.jsonUtils,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,re,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,time,header,Yes -gateway.modules.aichat.serviceGeneration.paths.codePath,typing,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,copy,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,json,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,logging,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelExtraction,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,time,header,Yes -gateway.modules.aichat.serviceGeneration.paths.documentPath,typing,header,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,base64,function generateImages,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,json,function generateImages,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,logging,header,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,time,header,Yes -gateway.modules.aichat.serviceGeneration.paths.imagePath,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,abc,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,logging,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.codeRendererBaseTemplate,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,PIL,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,abc,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,datetime,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,json,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,logging,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,re,function _determineFilename,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,threading,function _getAiStyles,Yes -gateway.modules.aichat.serviceGeneration.renderers.documentRendererBaseTemplate,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,importlib,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,logging,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,os,function discoverRenderers,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,pathlib,function discoverRenderers,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,sys,function discoverRenderers,Yes -gateway.modules.aichat.serviceGeneration.renderers.registry,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,(relative) .rendererCsv,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,csv,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeCsv,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,(relative) .rendererJson,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,json,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeJson,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,(relative) .codeRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,xml.dom,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCodeXml,xml.etree.ElementTree,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,csv,function _convertRowsToCsv,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,io,function _convertRowsToCsv,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererCsv,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,(relative) .rendererHtml,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,PIL,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,csv,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _setupDocumentStyles,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _createStyle,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.table,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.enum.text,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.ns,function _renderTableFastXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _renderTableFastXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableBordersXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableRowXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _applyHorizontalBordersOnly,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackground,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackgroundFast,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,docx.shared,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,lxml,function _renderTableFastXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,re,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _generateDocxFromJson,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderJsonTable,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,time,function _renderTableFastXml,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,base64,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,base64,function _replaceImageDataUris,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,html,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,html,function _replaceImageDataUris,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,re,function _replaceImageDataUris,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,re,function _extractImages,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,json,function _generateAiImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,logging,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _generateAiImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _compressPromptWithAi,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererImage,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererJson,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererJson,json,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererJson,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,(relative) .rendererHtml,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,PIL,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,base64,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,io,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,json,function _getAiStylesWithPdfColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelAi,function _getAiStylesWithPdfColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,re,function _getAiStylesWithPdfColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,re,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.enums,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.styles,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,reportlab.platypus,function _renderJsonImage,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,datetime,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,json,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,logging,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addTableToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addBulletListToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addHeadingToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addParagraphToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addCodeBlockToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSlideContentWithFrames,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderTextSectionsInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSectionToTextFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addTableToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addBulletListToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addParagraphToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSlideContentWithFrames,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderTextSectionsInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSectionToTextFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addTableToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addHeadingToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addParagraphToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addCodeBlockToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderTextSectionsInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSectionToTextFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlideInFrame,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,re,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,re,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,traceback,function _addImagesToSlide,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererText,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererText,typing,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .documentRendererBaseTemplate,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,(relative) .rendererCsv,function render,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,base64,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,base64,function _addImageToExcel,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,datetime,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,dateutil,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,io,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,io,function _addImageToExcel,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,json,function _getAiStylesWithExcelColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelAi,function _getAiStylesWithExcelColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelDocument,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.drawing.image,function _addImageToExcel,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.styles,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.utils,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,openpyxl.worksheet.table,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,re,header,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,re,function _getAiStylesWithExcelColors,Yes -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,asyncio,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,base64,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,base64,function _generateImageSection,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,json,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,logging,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.aichat.serviceGeneration.subContentIntegrator,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateSimpleSection,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateImageSection,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.shared.jsonUtils,function _generateSimpleSection,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,re,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,traceback,header,Yes -gateway.modules.aichat.serviceGeneration.subContentGenerator,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subContentIntegrator,json,function integrateContent,Yes -gateway.modules.aichat.serviceGeneration.subContentIntegrator,logging,header,Yes -gateway.modules.aichat.serviceGeneration.subContentIntegrator,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,json,header,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,logging,header,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,os,header,Yes -gateway.modules.aichat.serviceGeneration.subDocumentUtility,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subJsonSchema,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,logging,header,Yes -gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,modules.datamodels.datamodelJson,header,Yes -gateway.modules.aichat.serviceGeneration.subPromptBuilderGeneration,typing,header,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,json,header,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,json,function _createStructurePrompt,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,logging,header,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelAi,function generateStructure,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelJson,header,Yes -gateway.modules.aichat.serviceGeneration.subStructureGenerator,typing,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,asyncio,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,json,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,logging,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,modules.datamodels.datamodelAi,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,time,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,time,function _processCrawlResultsWithHierarchy,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,typing,header,Yes -gateway.modules.aichat.serviceWeb.mainServiceWeb,urllib.parse,header,Yes +gateway.modules.aicore.aicoreBase,abc,header,Yes +gateway.modules.aicore.aicoreBase,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicoreBase,time,function getCachedModels,Yes +gateway.modules.aicore.aicoreBase,typing,header,Yes +gateway.modules.aicore.aicoreModelRegistry,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicoreModelRegistry,importlib,header,Yes +gateway.modules.aicore.aicoreModelRegistry,logging,header,Yes +gateway.modules.aicore.aicoreModelRegistry,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.aicore.aicoreModelRegistry,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicoreModelRegistry,modules.datamodels.datamodelUam,header,Yes +gateway.modules.aicore.aicoreModelRegistry,modules.security.rbac,header,Yes +gateway.modules.aicore.aicoreModelRegistry,modules.security.rbacHelpers,header,Yes +gateway.modules.aicore.aicoreModelRegistry,os,header,Yes +gateway.modules.aicore.aicoreModelRegistry,time,function refreshModels,Yes +gateway.modules.aicore.aicoreModelRegistry,typing,header,Yes +gateway.modules.aicore.aicoreModelSelector,logging,header,Yes +gateway.modules.aicore.aicoreModelSelector,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicoreModelSelector,typing,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,base64,function callAiImage,Yes +gateway.modules.aicore.aicorePluginAnthropic,fastapi,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,httpx,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,logging,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,modules.shared.configuration,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,os,header,Yes +gateway.modules.aicore.aicorePluginAnthropic,time,function callAiImage,Yes +gateway.modules.aicore.aicorePluginAnthropic,typing,header,Yes +gateway.modules.aicore.aicorePluginInternal,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicorePluginInternal,logging,header,Yes +gateway.modules.aicore.aicorePluginInternal,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicorePluginInternal,typing,header,Yes +gateway.modules.aicore.aicorePluginOpenai,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicorePluginOpenai,fastapi,header,Yes +gateway.modules.aicore.aicorePluginOpenai,httpx,header,Yes +gateway.modules.aicore.aicorePluginOpenai,json,function generateImage,Yes +gateway.modules.aicore.aicorePluginOpenai,logging,header,Yes +gateway.modules.aicore.aicorePluginOpenai,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicorePluginOpenai,modules.shared.configuration,header,Yes +gateway.modules.aicore.aicorePluginOpenai,typing,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,fastapi,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,httpx,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,json,function webSearch,Yes +gateway.modules.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.aicore.aicorePluginPerplexity,json,function webCrawl,Yes +gateway.modules.aicore.aicorePluginPerplexity,logging,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,modules.datamodels.datamodelTools,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,modules.shared.configuration,header,Yes +gateway.modules.aicore.aicorePluginPerplexity,typing,header,Yes +gateway.modules.aicore.aicorePluginTavily,(relative) .aicoreBase,header,Yes +gateway.modules.aicore.aicorePluginTavily,asyncio,header,Yes +gateway.modules.aicore.aicorePluginTavily,dataclasses,header,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webSearch,Yes +gateway.modules.aicore.aicorePluginTavily,json,function webCrawl,Yes +gateway.modules.aicore.aicorePluginTavily,logging,header,Yes +gateway.modules.aicore.aicorePluginTavily,modules.datamodels.datamodelAi,header,Yes +gateway.modules.aicore.aicorePluginTavily,modules.datamodels.datamodelTools,header,Yes +gateway.modules.aicore.aicorePluginTavily,modules.shared.configuration,header,Yes +gateway.modules.aicore.aicorePluginTavily,re,header,Yes +gateway.modules.aicore.aicorePluginTavily,re,function _cleanUrl,Yes +gateway.modules.aicore.aicorePluginTavily,tavily,header,Yes +gateway.modules.aicore.aicorePluginTavily,typing,header,Yes +gateway.modules.aicore.aicorePluginTavily,urllib.parse,function _normalizeUrl,Yes gateway.modules.auth.__init__,(relative) .authentication,header,Yes gateway.modules.auth.__init__,(relative) .csrf,header,Yes gateway.modules.auth.__init__,(relative) .jwtService,header,Yes @@ -845,6 +241,13 @@ gateway.modules.datamodels.datamodelAudit,modules.shared.timeUtils,header,Yes gateway.modules.datamodels.datamodelAudit,pydantic,header,Yes gateway.modules.datamodels.datamodelAudit,typing,header,Yes gateway.modules.datamodels.datamodelAudit,uuid,header,Yes +gateway.modules.datamodels.datamodelChat,enum,header,Yes +gateway.modules.datamodels.datamodelChat,modules.datamodels.datamodelWorkflow,function updateFromSelection,Yes +gateway.modules.datamodels.datamodelChat,modules.shared.attributeUtils,header,Yes +gateway.modules.datamodels.datamodelChat,modules.shared.timeUtils,header,Yes +gateway.modules.datamodels.datamodelChat,pydantic,header,Yes +gateway.modules.datamodels.datamodelChat,typing,header,Yes +gateway.modules.datamodels.datamodelChat,uuid,header,Yes gateway.modules.datamodels.datamodelDocref,modules.shared.attributeUtils,header,Yes gateway.modules.datamodels.datamodelDocref,pydantic,header,Yes gateway.modules.datamodels.datamodelDocref,typing,header,Yes @@ -919,11 +322,15 @@ gateway.modules.datamodels.datamodelWorkflow,modules.shared.attributeUtils,heade gateway.modules.datamodels.datamodelWorkflow,modules.shared.jsonUtils,header,Yes gateway.modules.datamodels.datamodelWorkflow,pydantic,header,Yes gateway.modules.datamodels.datamodelWorkflow,typing,header,Yes -gateway.modules.datamodels.datamodelWorkflowActions,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.datamodels.datamodelWorkflowActions,modules.datamodels.datamodelChat,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.attributeUtils,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.frontendTypes,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,pydantic,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,typing,header,Yes +gateway.modules.features.automation.datamodelFeatureAutomation,modules.shared.attributeUtils,header,Yes +gateway.modules.features.automation.datamodelFeatureAutomation,pydantic,header,Yes +gateway.modules.features.automation.datamodelFeatureAutomation,typing,header,Yes +gateway.modules.features.automation.datamodelFeatureAutomation,uuid,header,Yes gateway.modules.features.automation.mainAutomation,logging,header,Yes gateway.modules.features.automation.mainAutomation,typing,header,Yes gateway.modules.features.automation.routeFeatureAutomation,(relative) .subAutomationTemplates,header,Yes @@ -933,10 +340,11 @@ gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,hea gateway.modules.features.automation.routeFeatureAutomation,fastapi.responses,function get_automations,Yes gateway.modules.features.automation.routeFeatureAutomation,json,header,Yes gateway.modules.features.automation.routeFeatureAutomation,logging,header,Yes -gateway.modules.features.automation.routeFeatureAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.automation.routeFeatureAutomation,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.auth,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.datamodels.datamodelChat,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.features.automation.datamodelFeatureAutomation,header,Yes +gateway.modules.features.automation.routeFeatureAutomation,modules.interfaces.interfaceDbChat,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.services,function execute_automation,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.shared.attributeUtils,header,Yes gateway.modules.features.automation.routeFeatureAutomation,modules.workflows.automation,header,Yes @@ -973,6 +381,7 @@ gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.data gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelRbac,header,Yes gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelUam,header,Yes gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.datamodels.datamodelUam,header,Yes +gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.features.automation.datamodelFeatureAutomation,header,Yes gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.features.chatbot.eventManager,function createMessage,Yes gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.features.chatbot.eventManager,function createLog,Yes gateway.modules.features.chatbot.interfaceFeatureChatbot,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes @@ -992,14 +401,12 @@ gateway.modules.features.chatbot.mainChatbot,asyncio,header,Yes gateway.modules.features.chatbot.mainChatbot,base64,header,Yes gateway.modules.features.chatbot.mainChatbot,json,header,Yes gateway.modules.features.chatbot.mainChatbot,logging,header,Yes -gateway.modules.features.chatbot.mainChatbot,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.features.chatbot.mainChatbot,modules.aichat.datamodelFeatureAiChat,function _emit_log_and_event,Yes gateway.modules.features.chatbot.mainChatbot,modules.connectors.connectorPreprocessor,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelAi,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelDocref,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.datamodels.datamodelUam,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,header,Yes -gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.chatbotConstants,function _processChatbotMessage,Yes +gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.datamodelFeatureChatbot,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.features.chatbot.eventManager,header,Yes gateway.modules.features.chatbot.mainChatbot,modules.interfaces.interfaceRbac,function _convert_file_ids_to_document_references,Yes gateway.modules.features.chatbot.mainChatbot,modules.services,header,Yes @@ -1020,7 +427,6 @@ gateway.modules.features.chatbot.routeFeatureChatbot,logging,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,math,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.auth,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.datamodels.datamodelPagination,header,Yes -gateway.modules.features.chatbot.routeFeatureChatbot,modules.datamodels.datamodelPagination,function get_chatbot_threads,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.interfaces.interfaceRbac,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.shared.timeUtils,header,Yes gateway.modules.features.chatbot.routeFeatureChatbot,modules.workflows.automation,header,Yes @@ -1044,7 +450,6 @@ gateway.modules.features.neutralizer.mainNeutralizePlayground,(relative) .datamo gateway.modules.features.neutralizer.mainNeutralizePlayground,asyncio,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,logging,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.datamodels.datamodelUam,header,Yes -gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.datamodels.datamodelUam,function _getSharepointConnection,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.services,header,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,modules.services.serviceSharepoint.mainServiceSharepoint,function processSharepointFiles,Yes gateway.modules.features.neutralizer.mainNeutralizePlayground,typing,header,Yes @@ -1203,8 +608,8 @@ gateway.modules.interfaces.interfaceAiObjects,asyncio,header,Yes gateway.modules.interfaces.interfaceAiObjects,base64,header,Yes gateway.modules.interfaces.interfaceAiObjects,dataclasses,header,Yes gateway.modules.interfaces.interfaceAiObjects,logging,header,Yes -gateway.modules.interfaces.interfaceAiObjects,modules.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.modules.interfaces.interfaceAiObjects,modules.aichat.aicore.aicoreModelSelector,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.aicore.aicoreModelRegistry,header,Yes +gateway.modules.interfaces.interfaceAiObjects,modules.aicore.aicoreModelSelector,header,Yes gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelAi,header,Yes gateway.modules.interfaces.interfaceAiObjects,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.interfaces.interfaceAiObjects,time,header,Yes @@ -1240,6 +645,30 @@ gateway.modules.interfaces.interfaceDbApp,modules.shared.timeUtils,header,Yes gateway.modules.interfaces.interfaceDbApp,passlib.context,header,Yes gateway.modules.interfaces.interfaceDbApp,typing,header,Yes gateway.modules.interfaces.interfaceDbApp,uuid,header,Yes +gateway.modules.interfaces.interfaceDbChat,asyncio,header,Yes +gateway.modules.interfaces.interfaceDbChat,datetime,function storeDebugMessageAndDocuments,Yes +gateway.modules.interfaces.interfaceDbChat,json,header,Yes +gateway.modules.interfaces.interfaceDbChat,logging,header,Yes +gateway.modules.interfaces.interfaceDbChat,math,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.connectors.connectorDbPostgre,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.datamodels.datamodelChat,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.datamodels.datamodelPagination,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.datamodels.datamodelRbac,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.datamodels.datamodelUam,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.features.automation.datamodelFeatureAutomation,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.interfaces.interfaceDbApp,function _enrichAutomationsWithUserAndMandate,Yes +gateway.modules.interfaces.interfaceDbChat,modules.interfaces.interfaceDbManagement,function storeDebugMessageAndDocuments,Yes +gateway.modules.interfaces.interfaceDbChat,modules.interfaces.interfaceRbac,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.security.rbac,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.security.rootAccess,function setUserContext,Yes +gateway.modules.interfaces.interfaceDbChat,modules.shared.callbackRegistry,function _notifyAutomationChanged,Yes +gateway.modules.interfaces.interfaceDbChat,modules.shared.configuration,header,Yes +gateway.modules.interfaces.interfaceDbChat,modules.shared.debugLogger,function storeDebugMessageAndDocuments,Yes +gateway.modules.interfaces.interfaceDbChat,modules.shared.timeUtils,header,Yes +gateway.modules.interfaces.interfaceDbChat,os,function storeDebugMessageAndDocuments,Yes +gateway.modules.interfaces.interfaceDbChat,typing,header,Yes +gateway.modules.interfaces.interfaceDbChat,uuid,header,Yes gateway.modules.interfaces.interfaceDbManagement,base64,header,Yes gateway.modules.interfaces.interfaceDbManagement,hashlib,header,Yes gateway.modules.interfaces.interfaceDbManagement,logging,header,Yes @@ -1309,11 +738,10 @@ gateway.modules.routes.routeAdmin,typing,header,Yes gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes gateway.modules.routes.routeAdminAutomationEvents,fastapi,header,Yes gateway.modules.routes.routeAdminAutomationEvents,logging,header,Yes -gateway.modules.routes.routeAdminAutomationEvents,modules.aichat.interfaceFeatureAiChat,header,Yes -gateway.modules.routes.routeAdminAutomationEvents,modules.aichat.interfaceFeatureAiChat,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.auth,header,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.datamodels.datamodelUam,header,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.interfaces.interfaceDbApp,function sync_all_automation_events,Yes +gateway.modules.routes.routeAdminAutomationEvents,modules.interfaces.interfaceDbChat,header,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.services,function sync_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.shared.eventManagement,function get_all_automation_events,Yes gateway.modules.routes.routeAdminAutomationEvents,modules.shared.eventManagement,function remove_event,Yes @@ -1336,8 +764,6 @@ gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,funct gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function getFeatureInstanceAvailableRoles,Yes gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelRbac,function _hasMandateAdminRole,Yes gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,function listFeatureInstanceUsers,Yes -gateway.modules.routes.routeAdminFeatures,modules.datamodels.datamodelUam,function addUserToFeatureInstance,Yes gateway.modules.routes.routeAdminFeatures,modules.interfaces.interfaceDbApp,header,Yes gateway.modules.routes.routeAdminFeatures,modules.interfaces.interfaceFeatures,header,Yes gateway.modules.routes.routeAdminFeatures,pydantic,header,Yes @@ -1358,10 +784,8 @@ gateway.modules.routes.routeAdminRbacRoles,fastapi,header,Yes gateway.modules.routes.routeAdminRbacRoles,logging,header,Yes gateway.modules.routes.routeAdminRbacRoles,modules.auth,header,Yes gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelMembership,header,Yes -gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelMembership,function listUsersWithRoles,Yes gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelRbac,header,Yes gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeAdminRbacRoles,modules.datamodels.datamodelUam,function listUsersWithRoles,Yes gateway.modules.routes.routeAdminRbacRoles,modules.interfaces.interfaceDbApp,header,Yes gateway.modules.routes.routeAdminRbacRoles,typing,header,Yes gateway.modules.routes.routeAdminRbacRules,fastapi,header,Yes @@ -1379,13 +803,19 @@ gateway.modules.routes.routeAttributes,fastapi,header,Yes gateway.modules.routes.routeAttributes,logging,header,Yes gateway.modules.routes.routeAttributes,modules.auth,header,Yes gateway.modules.routes.routeAttributes,modules.shared.attributeUtils,header,Yes +gateway.modules.routes.routeChat,(relative) .,header,Yes +gateway.modules.routes.routeChat,fastapi,header,Yes +gateway.modules.routes.routeChat,logging,header,Yes +gateway.modules.routes.routeChat,modules.auth,header,Yes +gateway.modules.routes.routeChat,modules.datamodels.datamodelChat,header,Yes +gateway.modules.routes.routeChat,modules.workflows.automation,header,Yes +gateway.modules.routes.routeChat,typing,header,Yes gateway.modules.routes.routeDataConnections,fastapi,header,Yes gateway.modules.routes.routeDataConnections,fastapi,header,Yes gateway.modules.routes.routeDataConnections,json,header,Yes gateway.modules.routes.routeDataConnections,logging,header,Yes gateway.modules.routes.routeDataConnections,math,header,Yes gateway.modules.routes.routeDataConnections,modules.auth,header,Yes -gateway.modules.routes.routeDataConnections,modules.auth,function get_connections,Yes gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelPagination,header,Yes gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelSecurity,header,Yes gateway.modules.routes.routeDataConnections,modules.datamodels.datamodelUam,header,Yes @@ -1411,7 +841,6 @@ gateway.modules.routes.routeDataMandates,json,header,Yes gateway.modules.routes.routeDataMandates,logging,header,Yes gateway.modules.routes.routeDataMandates,modules.auth,header,Yes gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelMembership,header,Yes -gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelMembership,function delete_mandate,Yes gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelPagination,header,Yes gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelRbac,header,Yes gateway.modules.routes.routeDataMandates,modules.datamodels.datamodelUam,header,Yes @@ -1444,12 +873,7 @@ gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,fun gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelMembership,function sendPasswordLink,Yes gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelPagination,header,Yes gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function create_user,Yes -gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function reset_user_password,Yes -gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function change_password,Yes -gateway.modules.routes.routeDataUsers,modules.datamodels.datamodelUam,function get_users,Yes gateway.modules.routes.routeDataUsers,modules.interfaces.interfaceDbApp,header,Yes -gateway.modules.routes.routeDataUsers,modules.interfaces.interfaceDbApp,function sendPasswordLink,Yes gateway.modules.routes.routeDataUsers,modules.services,function sendPasswordLink,Yes gateway.modules.routes.routeDataUsers,modules.shared.auditLogger,function reset_user_password,Yes gateway.modules.routes.routeDataUsers,modules.shared.auditLogger,function change_password,Yes @@ -1460,12 +884,12 @@ gateway.modules.routes.routeDataUsers,typing,header,Yes gateway.modules.routes.routeDataWorkflows,fastapi,header,Yes gateway.modules.routes.routeDataWorkflows,json,header,Yes gateway.modules.routes.routeDataWorkflows,logging,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.aichat.interfaceFeatureAiChat,header,Yes -gateway.modules.routes.routeDataWorkflows,modules.aichat.interfaceFeatureAiChat,header,Yes gateway.modules.routes.routeDataWorkflows,modules.auth,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelChat,header,Yes gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelPagination,header,Yes gateway.modules.routes.routeDataWorkflows,modules.datamodels.datamodelUam,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.interfaces.interfaceDbChat,header,Yes +gateway.modules.routes.routeDataWorkflows,modules.interfaces.interfaceDbChat,header,Yes gateway.modules.routes.routeDataWorkflows,modules.interfaces.interfaceRbac,header,Yes gateway.modules.routes.routeDataWorkflows,modules.services,function get_all_actions,Yes gateway.modules.routes.routeDataWorkflows,modules.services,function get_method_actions,Yes @@ -1495,9 +919,6 @@ gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function gateway.modules.routes.routeGdpr,modules.datamodels.datamodelMembership,function deleteAccount,Yes gateway.modules.routes.routeGdpr,modules.datamodels.datamodelSecurity,function deleteAccount,Yes gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function deleteAccount,Yes -gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function exportUserData,Yes -gateway.modules.routes.routeGdpr,modules.datamodels.datamodelUam,function exportPortableData,Yes gateway.modules.routes.routeGdpr,modules.interfaces.interfaceDbApp,header,Yes gateway.modules.routes.routeGdpr,modules.shared.auditLogger,header,Yes gateway.modules.routes.routeGdpr,modules.shared.timeUtils,header,Yes @@ -1553,13 +974,8 @@ gateway.modules.routes.routeSecurityGoogle,json,header,Yes gateway.modules.routes.routeSecurityGoogle,logging,header,Yes gateway.modules.routes.routeSecurityGoogle,modules.auth,header,Yes gateway.modules.routes.routeSecurityGoogle,modules.auth,header,Yes -gateway.modules.routes.routeSecurityGoogle,modules.auth,function verify_token,Yes -gateway.modules.routes.routeSecurityGoogle,modules.auth,function refresh_token,Yes -gateway.modules.routes.routeSecurityGoogle,modules.auth,function auth_callback,Yes gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelSecurity,function auth_callback,Yes gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,function login,Yes -gateway.modules.routes.routeSecurityGoogle,modules.datamodels.datamodelUam,function auth_callback,Yes gateway.modules.routes.routeSecurityGoogle,modules.interfaces.interfaceDbApp,header,Yes gateway.modules.routes.routeSecurityGoogle,modules.shared.auditLogger,function logout,Yes gateway.modules.routes.routeSecurityGoogle,modules.shared.configuration,header,Yes @@ -1578,8 +994,6 @@ gateway.modules.routes.routeSecurityLocal,modules.auth,header,Yes gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelMessaging,function _sendAuthEmail,Yes gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelSecurity,header,Yes gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,header,Yes -gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,function login,Yes -gateway.modules.routes.routeSecurityLocal,modules.datamodels.datamodelUam,function register_user,Yes gateway.modules.routes.routeSecurityLocal,modules.interfaces.interfaceDbApp,header,Yes gateway.modules.routes.routeSecurityLocal,modules.interfaces.interfaceMessaging,function _sendAuthEmail,Yes gateway.modules.routes.routeSecurityLocal,modules.shared.auditLogger,function login,Yes @@ -1597,9 +1011,6 @@ gateway.modules.routes.routeSecurityMsft,json,header,Yes gateway.modules.routes.routeSecurityMsft,logging,header,Yes gateway.modules.routes.routeSecurityMsft,modules.auth,header,Yes gateway.modules.routes.routeSecurityMsft,modules.auth,header,Yes -gateway.modules.routes.routeSecurityMsft,modules.auth,function refresh_token,Yes -gateway.modules.routes.routeSecurityMsft,modules.auth,function refresh_token,Yes -gateway.modules.routes.routeSecurityMsft,modules.auth,function auth_callback,Yes gateway.modules.routes.routeSecurityMsft,modules.datamodels.datamodelSecurity,header,Yes gateway.modules.routes.routeSecurityMsft,modules.datamodels.datamodelUam,header,Yes gateway.modules.routes.routeSecurityMsft,modules.interfaces.interfaceDbApp,header,Yes @@ -1648,30 +1059,576 @@ gateway.modules.security.rootAccess,modules.connectors.connectorDbPostgre,header gateway.modules.security.rootAccess,modules.datamodels.datamodelUam,header,Yes gateway.modules.security.rootAccess,modules.interfaces.interfaceBootstrap,function _ensureBootstrap,Yes gateway.modules.security.rootAccess,modules.shared.configuration,header,Yes +gateway.modules.services.__init__,(relative) .serviceAi.mainServiceAi,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceChat.mainServiceChat,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceExtraction.mainServiceExtraction,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceGeneration.mainServiceGeneration,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceMessaging.mainServiceMessaging,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceSecurity.mainServiceSecurity,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceSharepoint.mainServiceSharepoint,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceTicket.mainServiceTicket,function __init__,Yes gateway.modules.services.__init__,(relative) .serviceUtils.mainServiceUtils,function __init__,Yes +gateway.modules.services.__init__,(relative) .serviceWeb.mainServiceWeb,function __init__,Yes gateway.modules.services.__init__,glob,header,Yes gateway.modules.services.__init__,importlib,header,Yes gateway.modules.services.__init__,logging,header,Yes -gateway.modules.services.__init__,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.services.__init__,modules.datamodels.datamodelChat,header,Yes gateway.modules.services.__init__,modules.datamodels.datamodelUam,header,Yes gateway.modules.services.__init__,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.modules.services.__init__,modules.interfaces.interfaceDbChat,function __init__,Yes gateway.modules.services.__init__,modules.interfaces.interfaceDbManagement,function __init__,Yes gateway.modules.services.__init__,os,header,Yes gateway.modules.services.__init__,typing,header,Yes +gateway.modules.services.serviceAi.mainAiChat,logging,header,Yes +gateway.modules.services.serviceAi.mainAiChat,modules.aicore.aicoreModelRegistry,function onStart,Yes +gateway.modules.services.serviceAi.mainAiChat,typing,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subAiCallLooping,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subContentExtraction,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subDocumentIntents,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subResponseParsing,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subStructureFilling,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,(relative) .subStructureGeneration,function _initializeSubmodules,Yes +gateway.modules.services.serviceAi.mainServiceAi,base64,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,json,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,logging,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelChat,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.interfaces.interfaceAiObjects,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.services.serviceExtraction.mainServiceExtraction,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.services.serviceGeneration.mainServiceGeneration,function renderResult,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.services.serviceGeneration.paths.codePath,function _handleCodeGeneration,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.services.serviceGeneration.paths.documentPath,function _handleDocumentGeneration,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.services.serviceGeneration.paths.imagePath,function _handleImageGeneration,Yes +gateway.modules.services.serviceAi.mainServiceAi,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,re,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,time,header,Yes +gateway.modules.services.serviceAi.mainServiceAi,time,function _handleDataExtraction,Yes +gateway.modules.services.serviceAi.mainServiceAi,typing,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,(relative) .subLoopingUseCases,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,json,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,logging,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.shared.jsonContinuation,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceAi.subAiCallLooping,typing,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,base64,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,json,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,logging,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function extractTextFromImage,Yes +gateway.modules.services.serviceAi.subContentExtraction,modules.datamodels.datamodelAi,function processTextContentWithAi,Yes +gateway.modules.services.serviceAi.subContentExtraction,modules.datamodels.datamodelChat,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceAi.subContentExtraction,traceback,function extractTextFromImage,Yes +gateway.modules.services.serviceAi.subContentExtraction,traceback,function processTextContentWithAi,Yes +gateway.modules.services.serviceAi.subContentExtraction,typing,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,json,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,logging,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,modules.datamodels.datamodelChat,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceAi.subDocumentIntents,traceback,function resolvePreExtractedDocument,Yes +gateway.modules.services.serviceAi.subDocumentIntents,typing,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,datetime,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,json,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,logging,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,os,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,re,header,Yes +gateway.modules.services.serviceAi.subJsonMerger,typing,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,(relative) .subJsonMerger,function mergeJsonStringsWithOverlap,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,json,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,logging,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,modules.shared.debugLogger,function mergeFragmentIntoSection,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,re,header,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,re,function _extractRowsFromFragment,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,re,function _detectAndNormalizeFragment,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,traceback,function _mergeJsonStructuresGeneric,Yes +gateway.modules.services.serviceAi.subJsonResponseHandling,typing,header,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,dataclasses,header,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,json,function _handleChapterStructureFinalResult,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,json,function _handleCodeStructureFinalResult,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,json,function _handleCodeContentFinalResult,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,logging,header,Yes +gateway.modules.services.serviceAi.subLoopingUseCases,typing,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,(relative) .subJsonResponseHandling,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,json,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,logging,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceAi.subResponseParsing,typing,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,asyncio,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,base64,function _processAiResponseForSection,Yes +gateway.modules.services.serviceAi.subStructureFilling,copy,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,json,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,logging,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.datamodels.datamodelJson,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.services.serviceGeneration.renderers.registry,function _getAcceptedSectionTypesForFormat,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.shared.jsonContinuation,function buildSectionPromptWithContinuation,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _extractAndMergeMultipleJsonBlocks,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processAiResponseForSection,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.shared.jsonUtils,function _processSingleSection,Yes +gateway.modules.services.serviceAi.subStructureFilling,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceAi.subStructureFilling,typing,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,json,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,logging,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.services.serviceGeneration.renderers.registry,function generateStructure,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.shared,function generateStructure,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.shared.jsonContinuation,function generateStructure,Yes +gateway.modules.services.serviceAi.subStructureGeneration,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceAi.subStructureGeneration,typing,header,Yes gateway.modules.services.serviceChat.mainServiceChat,json,function calculateObjectSize,Yes gateway.modules.services.serviceChat.mainServiceChat,logging,header,Yes -gateway.modules.services.serviceChat.mainServiceChat,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelChat,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelDocref,function getChatDocumentsFromDocumentList,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.datamodels.datamodelUam,header,Yes gateway.modules.services.serviceChat.mainServiceChat,modules.shared.progressLogger,header,Yes gateway.modules.services.serviceChat.mainServiceChat,sys,function calculateObjectSize,Yes gateway.modules.services.serviceChat.mainServiceChat,typing,header,Yes +gateway.modules.services.serviceExtraction.__init__,(relative) .mainServiceExtraction,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,PIL,function chunk,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,base64,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,io,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerImage,typing,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerStructure,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerStructure,json,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerStructure,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerStructure,typing,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerTable,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerTable,typing,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerText,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerText,logging,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.chunking.chunkerText,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorBinary,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorBinary,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorBinary,base64,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorBinary,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorBinary,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorCsv,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorCsv,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorCsv,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorCsv,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,docx,function _load,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,io,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorDocx,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorHtml,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorHtml,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorHtml,bs4,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorHtml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorHtml,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,PIL,function extract,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,base64,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,io,function extract,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,logging,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorImage,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorJson,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorJson,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorJson,json,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorJson,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorJson,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,PyPDF2,function _load,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,base64,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,fitz,function _load,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,io,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPdf,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,base64,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,io,function extract,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,logging,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,pptx,function _load,Yes +gateway.modules.services.serviceExtraction.extractors.extractorPptx,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorSql,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorSql,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorSql,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorSql,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorText,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorText,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorText,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,datetime,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,io,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,openpyxl,function _load,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXlsx,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXml,(relative) ..subRegistry,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXml,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXml,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXml,typing,header,Yes +gateway.modules.services.serviceExtraction.extractors.extractorXml,xml.etree.ElementTree,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerDefault,function applyMerging,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerTable,function applyMerging,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .merging.mergerText,function applyMerging,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .subMerger,function applyMerging,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .subPipeline,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,(relative) .subRegistry,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,asyncio,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,base64,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,json,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,logging,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.aicore.aicoreModelRegistry,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.aicore.aicoreModelSelector,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelChat,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.interfaces.interfaceDbManagement,function extractContent,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.shared.debugLogger,function extractContent,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,time,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,typing,header,Yes +gateway.modules.services.serviceExtraction.mainServiceExtraction,uuid,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerDefault,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerDefault,typing,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerTable,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerTable,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerTable,typing,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerText,(relative) ..subUtils,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerText,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.merging.mergerText,typing,header,Yes +gateway.modules.services.serviceExtraction.subMerger,(relative) .subUtils,header,Yes +gateway.modules.services.serviceExtraction.subMerger,logging,header,Yes +gateway.modules.services.serviceExtraction.subMerger,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.subMerger,typing,header,Yes +gateway.modules.services.serviceExtraction.subPipeline,(relative) .mainServiceExtraction,function runExtraction,Yes +gateway.modules.services.serviceExtraction.subPipeline,(relative) .subRegistry,header,Yes +gateway.modules.services.serviceExtraction.subPipeline,(relative) .subUtils,header,Yes +gateway.modules.services.serviceExtraction.subPipeline,logging,header,Yes +gateway.modules.services.serviceExtraction.subPipeline,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.subPipeline,typing,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,json,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,logging,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,modules.shared.debugLogger,function buildExtractionPrompt,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction,typing,header,Yes +gateway.modules.services.serviceExtraction.subRegistry,(relative) .chunking.chunkerImage,function __init__,Yes +gateway.modules.services.serviceExtraction.subRegistry,(relative) .chunking.chunkerStructure,function __init__,Yes +gateway.modules.services.serviceExtraction.subRegistry,(relative) .chunking.chunkerTable,function __init__,Yes +gateway.modules.services.serviceExtraction.subRegistry,(relative) .chunking.chunkerText,function __init__,Yes +gateway.modules.services.serviceExtraction.subRegistry,(relative) .extractors.extractorBinary,function _auto_discover_extractors,Yes +gateway.modules.services.serviceExtraction.subRegistry,importlib,function _auto_discover_extractors,Yes +gateway.modules.services.serviceExtraction.subRegistry,logging,header,Yes +gateway.modules.services.serviceExtraction.subRegistry,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceExtraction.subRegistry,os,function _auto_discover_extractors,Yes +gateway.modules.services.serviceExtraction.subRegistry,pathlib,function _auto_discover_extractors,Yes +gateway.modules.services.serviceExtraction.subRegistry,traceback,function _auto_discover_extractors,Yes +gateway.modules.services.serviceExtraction.subRegistry,traceback,function __init__,Yes +gateway.modules.services.serviceExtraction.subRegistry,typing,header,Yes +gateway.modules.services.serviceExtraction.subUtils,uuid,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,(relative) .renderers.registry,function _getFormatRenderer,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,base64,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,logging,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelChat,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.services.serviceExtraction.subPromptBuilderExtraction,function getAdaptiveExtractionPrompt,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.services.serviceGeneration.renderers.registry,function renderReport,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.services.serviceGeneration.subContentGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.services.serviceGeneration.subDocumentUtility,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,modules.services.serviceGeneration.subStructureGenerator,function generateDocumentWithTwoPhases,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,traceback,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,typing,header,Yes +gateway.modules.services.serviceGeneration.mainServiceGeneration,uuid,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,json,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,logging,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.datamodels.datamodelDocument,function generateCode,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.services.serviceGeneration.renderers.registry,function _getCodeRenderer,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateCodeStructure,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.shared.jsonContinuation,function _generateSingleFileContent,Yes +gateway.modules.services.serviceGeneration.paths.codePath,modules.shared.jsonUtils,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,re,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,time,header,Yes +gateway.modules.services.serviceGeneration.paths.codePath,typing,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,copy,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,json,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,logging,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,modules.datamodels.datamodelExtraction,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,time,header,Yes +gateway.modules.services.serviceGeneration.paths.documentPath,typing,header,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,base64,function generateImages,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,json,function generateImages,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,logging,header,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,modules.datamodels.datamodelWorkflow,header,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,time,header,Yes +gateway.modules.services.serviceGeneration.paths.imagePath,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.codeRendererBaseTemplate,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.codeRendererBaseTemplate,abc,header,Yes +gateway.modules.services.serviceGeneration.renderers.codeRendererBaseTemplate,logging,header,Yes +gateway.modules.services.serviceGeneration.renderers.codeRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.codeRendererBaseTemplate,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,PIL,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,abc,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,datetime,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,json,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,logging,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,modules.datamodels.datamodelJson,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,re,header,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,re,function _determineFilename,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,threading,function _getAiStyles,Yes +gateway.modules.services.serviceGeneration.renderers.documentRendererBaseTemplate,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.registry,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.registry,importlib,header,Yes +gateway.modules.services.serviceGeneration.renderers.registry,logging,header,Yes +gateway.modules.services.serviceGeneration.renderers.registry,os,function discoverRenderers,Yes +gateway.modules.services.serviceGeneration.renderers.registry,pathlib,function discoverRenderers,Yes +gateway.modules.services.serviceGeneration.renderers.registry,sys,function discoverRenderers,Yes +gateway.modules.services.serviceGeneration.renderers.registry,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,(relative) .rendererCsv,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,csv,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeCsv,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeJson,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeJson,(relative) .rendererJson,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeJson,json,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeJson,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeXml,(relative) .codeRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeXml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeXml,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeXml,xml.dom,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCodeXml,xml.etree.ElementTree,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCsv,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCsv,csv,function _convertRowsToCsv,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCsv,io,function _convertRowsToCsv,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCsv,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererCsv,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,(relative) .rendererHtml,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,PIL,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,csv,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _setupDocumentStyles,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.enum.style,function _createStyle,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.enum.table,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.enum.text,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.ns,function _renderTableFastXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _renderTableFastXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableBordersXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _createTableRowXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _applyHorizontalBordersOnly,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackground,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.oxml.shared,function _setCellBackgroundFast,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,docx.shared,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,lxml,function _renderTableFastXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,re,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,time,function _generateDocxFromJson,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,time,function _renderJsonTable,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,time,function _renderTableFastXml,Yes +gateway.modules.services.serviceGeneration.renderers.rendererDocx,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,base64,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,base64,function _replaceImageDataUris,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,html,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,html,function _replaceImageDataUris,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,re,function _replaceImageDataUris,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,re,function _extractImages,Yes +gateway.modules.services.serviceGeneration.renderers.rendererHtml,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,json,function _generateAiImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,logging,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _generateAiImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelAi,function _compressPromptWithAi,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererImage,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererJson,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererJson,json,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererJson,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererJson,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererMarkdown,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererMarkdown,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererMarkdown,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,(relative) .rendererHtml,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,PIL,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,base64,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,io,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,json,function _getAiStylesWithPdfColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelAi,function _getAiStylesWithPdfColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,re,function _getAiStylesWithPdfColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,re,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.enums,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.pagesizes,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.styles,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.lib.units,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.platypus,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,reportlab.platypus,function _renderJsonImage,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPdf,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,PIL,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,base64,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,datetime,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,io,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,json,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,logging,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addTableToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addBulletListToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addHeadingToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addParagraphToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _addCodeBlockToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSlideContentWithFrames,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderTextSectionsInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.dml.color,function _renderSectionToTextFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addTableToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addBulletListToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addParagraphToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSlideContentWithFrames,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderTextSectionsInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _renderSectionToTextFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.enum.text,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addTableToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addHeadingToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addParagraphToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addCodeBlockToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderTextSectionsInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSectionToTextFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addImagesToSlideInFrame,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _renderSlideContentWithFrames,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,pptx.util,function _addBulletListToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,re,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,re,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,traceback,function _addImagesToSlide,Yes +gateway.modules.services.serviceGeneration.renderers.rendererPptx,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererText,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererText,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererText,typing,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,(relative) .documentRendererBaseTemplate,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,(relative) .rendererCsv,function render,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,base64,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,base64,function _addImageToExcel,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,datetime,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,dateutil,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,io,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,io,function _addImageToExcel,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,json,function _getAiStylesWithExcelColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelAi,function _getAiStylesWithExcelColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelDocument,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,modules.datamodels.datamodelJson,function getAcceptedSectionTypes,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,openpyxl,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,openpyxl.drawing.image,function _addImageToExcel,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,openpyxl.styles,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,openpyxl.utils,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,openpyxl.worksheet.table,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,re,header,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,re,function _getAiStylesWithExcelColors,Yes +gateway.modules.services.serviceGeneration.renderers.rendererXlsx,typing,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,asyncio,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,base64,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,base64,function _generateImageSection,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,json,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,logging,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateSimpleSection,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,modules.datamodels.datamodelAi,function _generateImageSection,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,modules.services.serviceGeneration.subContentIntegrator,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,modules.shared.jsonUtils,function _generateSimpleSection,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,modules.workflows.processing.shared.stateTools,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,re,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,traceback,header,Yes +gateway.modules.services.serviceGeneration.subContentGenerator,typing,header,Yes +gateway.modules.services.serviceGeneration.subContentIntegrator,json,function integrateContent,Yes +gateway.modules.services.serviceGeneration.subContentIntegrator,logging,header,Yes +gateway.modules.services.serviceGeneration.subContentIntegrator,typing,header,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,csv,function convertDocumentDataToString,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,io,function convertDocumentDataToString,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,json,header,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,logging,header,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,os,header,Yes +gateway.modules.services.serviceGeneration.subDocumentUtility,typing,header,Yes +gateway.modules.services.serviceGeneration.subJsonSchema,typing,header,Yes +gateway.modules.services.serviceGeneration.subPromptBuilderGeneration,logging,header,Yes +gateway.modules.services.serviceGeneration.subPromptBuilderGeneration,modules.datamodels.datamodelJson,header,Yes +gateway.modules.services.serviceGeneration.subPromptBuilderGeneration,typing,header,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,json,header,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,json,function _createStructurePrompt,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,logging,header,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelAi,function generateStructure,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,modules.datamodels.datamodelJson,header,Yes +gateway.modules.services.serviceGeneration.subStructureGenerator,typing,header,Yes gateway.modules.services.serviceMessaging.mainServiceMessaging,html,function _textToHtml,Yes gateway.modules.services.serviceMessaging.mainServiceMessaging,importlib,function _loadSubscriptionFunction,Yes gateway.modules.services.serviceMessaging.mainServiceMessaging,logging,header,Yes @@ -1701,7 +1658,7 @@ gateway.modules.services.serviceTicket.mainServiceTicket,modules.interfaces.inte gateway.modules.services.serviceTicket.mainServiceTicket,typing,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,json,function writeDebugArtifact,Yes gateway.modules.services.serviceUtils.mainServiceUtils,logging,header,Yes -gateway.modules.services.serviceUtils.mainServiceUtils,modules.aichat.interfaceFeatureAiChat,function storeDebugMessageAndDocuments,Yes +gateway.modules.services.serviceUtils.mainServiceUtils,modules.interfaces.interfaceDbChat,function storeDebugMessageAndDocuments,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.configuration,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.debugLogger,function writeDebugFile,Yes @@ -1711,6 +1668,14 @@ gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.eventManag gateway.modules.services.serviceUtils.mainServiceUtils,modules.shared.timeUtils,header,Yes gateway.modules.services.serviceUtils.mainServiceUtils,re,function sanitizePromptContent,Yes gateway.modules.services.serviceUtils.mainServiceUtils,typing,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,asyncio,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,json,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,logging,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,modules.datamodels.datamodelAi,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,time,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,time,function _processCrawlResultsWithHierarchy,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,typing,header,Yes +gateway.modules.services.serviceWeb.mainServiceWeb,urllib.parse,header,Yes gateway.modules.shared.__init__,(relative) .,header,Yes gateway.modules.shared.__init__,(relative) .,header,Yes gateway.modules.shared.__init__,(relative) .,header,Yes @@ -1797,8 +1762,9 @@ gateway.modules.workflows.automation.__init__,(relative) .mainWorkflow,header,Ye gateway.modules.workflows.automation.mainWorkflow,(relative) .subAutomationUtils,header,Yes gateway.modules.workflows.automation.mainWorkflow,json,header,Yes gateway.modules.workflows.automation.mainWorkflow,logging,header,Yes -gateway.modules.workflows.automation.mainWorkflow,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.datamodels.datamodelUam,header,Yes +gateway.modules.workflows.automation.mainWorkflow,modules.features.automation.datamodelFeatureAutomation,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.services,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.shared.eventManagement,header,Yes gateway.modules.workflows.automation.mainWorkflow,modules.shared.timeUtils,header,Yes @@ -1821,11 +1787,11 @@ gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .summariz gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .translateDocument,header,Yes gateway.modules.workflows.methods.methodAi.actions.__init__,(relative) .webResearch,header,Yes gateway.modules.workflows.methods.methodAi.actions.convertDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.convertDocument,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.convertDocument,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.convertDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelDocref,function generateCode,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,modules.datamodels.datamodelWorkflow,header,Yes @@ -1833,8 +1799,8 @@ gateway.modules.workflows.methods.methodAi.actions.generateCode,re,function gene gateway.modules.workflows.methods.methodAi.actions.generateCode,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateCode,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelDocref,function generateDocument,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodAi.actions.generateDocument,modules.datamodels.datamodelWorkflow,header,Yes @@ -1843,10 +1809,8 @@ gateway.modules.workflows.methods.methodAi.actions.generateDocument,time,header, gateway.modules.workflows.methods.methodAi.actions.generateDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,json,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.process,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,header,Yes -gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes -gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelAi,function process,Yes +gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelDocref,function process,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelDocref,function process,Yes gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.datamodelExtraction,header,Yes @@ -1854,13 +1818,13 @@ gateway.modules.workflows.methods.methodAi.actions.process,modules.datamodels.da gateway.modules.workflows.methods.methodAi.actions.process,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.process,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.summarizeDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.translateDocument,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.translateDocument,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.translateDocument,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.translateDocument,typing,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,logging,header,Yes -gateway.modules.workflows.methods.methodAi.actions.webResearch,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodAi.actions.webResearch,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,re,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,time,header,Yes gateway.modules.workflows.methods.methodAi.actions.webResearch,typing,header,Yes @@ -1891,8 +1855,8 @@ gateway.modules.workflows.methods.methodBase,typing,header,Yes gateway.modules.workflows.methods.methodChatbot.__init__,(relative) .methodChatbot,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,json,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,logging,header,Yes -gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.aichat.datamodelFeatureAiChat,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.connectors.connectorPreprocessor,header,Yes +gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.datamodels.datamodelDocref,function queryDatabase,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,modules.workflows.methods.methodBase,header,Yes gateway.modules.workflows.methods.methodChatbot.actions.queryDatabase,time,header,Yes @@ -1908,17 +1872,17 @@ gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .get gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .neutralizeData,header,Yes gateway.modules.workflows.methods.methodContext.actions.__init__,(relative) .triggerPreprocessingServer,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelDocref,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,time,header,Yes gateway.modules.workflows.methods.methodContext.actions.extractContent,typing,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,json,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.getDocumentIndex,typing,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelDocref,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,modules.datamodels.datamodelExtraction,header,Yes gateway.modules.workflows.methods.methodContext.actions.neutralizeData,time,header,Yes @@ -1926,7 +1890,7 @@ gateway.modules.workflows.methods.methodContext.actions.neutralizeData,typing,he gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,aiohttp,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,json,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,logging,header,Yes -gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,modules.shared.configuration,header,Yes gateway.modules.workflows.methods.methodContext.actions.triggerPreprocessingServer,typing,header,Yes gateway.modules.workflows.methods.methodContext.helpers.documentIndex,datetime,header,Yes @@ -1955,7 +1919,7 @@ gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseC gateway.modules.workflows.methods.methodJira.actions.__init__,(relative) .parseExcelContent,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,modules.shared.configuration,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.connectJira,uuid,header,Yes @@ -1965,7 +1929,7 @@ gateway.modules.workflows.methods.methodJira.actions.createCsvContent,datetime,h gateway.modules.workflows.methods.methodJira.actions.createCsvContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.createCsvContent,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createCsvContent,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.createCsvContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,base64,header,Yes @@ -1974,31 +1938,31 @@ gateway.modules.workflows.methods.methodJira.actions.createExcelContent,datetime gateway.modules.workflows.methods.methodJira.actions.createExcelContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.createExcelContent,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.createExcelContent,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.createExcelContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.exportTicketsAsJson,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.importTicketsFromJson,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.mergeTicketData,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseCsvContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,io,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,json,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,logging,header,Yes -gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,pandas,header,Yes gateway.modules.workflows.methods.methodJira.actions.parseExcelContent,typing,header,Yes gateway.modules.workflows.methods.methodJira.helpers.adfConverter,logging,header,Yes @@ -2030,7 +1994,7 @@ gateway.modules.workflows.methods.methodOutlook.actions.__init__,(relative) .sen gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,base64,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,modules.datamodels.datamodelDocref,function composeAndDraftEmailWithContext,Yes @@ -2041,18 +2005,18 @@ gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWith gateway.modules.workflows.methods.methodOutlook.actions.composeAndDraftEmailWithContext,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.readEmails,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.readEmails,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,time,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.readEmails,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.searchEmails,typing,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,json,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,logging,header,Yes -gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,modules.datamodels.datamodelDocref,function sendDraftEmail,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,requests,header,Yes gateway.modules.workflows.methods.methodOutlook.actions.sendDraftEmail,time,header,Yes @@ -2091,53 +2055,53 @@ gateway.modules.workflows.methods.methodSharepoint.actions.__init__,(relative) . gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,datetime,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.analyzeFolderUsage,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,modules.datamodels.datamodelDocref,function copyFile,Yes gateway.modules.workflows.methods.methodSharepoint.actions.copyFile,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,modules.datamodels.datamodelDocref,function downloadFileByPath,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,os,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.downloadFileByPath,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findDocumentPath,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.findSiteByUrl,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.listDocuments,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.readDocuments,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,time,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,typing,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadDocument,urllib.parse,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,base64,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,json,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,logging,header,Yes -gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,modules.datamodels.datamodelDocref,function uploadFile,Yes gateway.modules.workflows.methods.methodSharepoint.actions.uploadFile,typing,header,Yes @@ -2198,23 +2162,22 @@ gateway.modules.workflows.processing.adaptive.progressTracker,datetime,header,Ye gateway.modules.workflows.processing.adaptive.progressTracker,logging,header,Yes gateway.modules.workflows.processing.adaptive.progressTracker,typing,header,Yes gateway.modules.workflows.processing.core.actionExecutor,logging,header,Yes -gateway.modules.workflows.processing.core.actionExecutor,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.actionExecutor,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.core.actionExecutor,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.core.messageCreator,function _createActionCompletionMessage,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.core.actionExecutor,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.actionExecutor,time,function executeSingleAction,Yes gateway.modules.workflows.processing.core.actionExecutor,typing,header,Yes gateway.modules.workflows.processing.core.messageCreator,logging,header,Yes -gateway.modules.workflows.processing.core.messageCreator,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.messageCreator,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.core.messageCreator,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.core.messageCreator,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.messageCreator,typing,header,Yes gateway.modules.workflows.processing.core.taskPlanner,json,header,Yes gateway.modules.workflows.processing.core.taskPlanner,logging,header,Yes -gateway.modules.workflows.processing.core.taskPlanner,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.core.taskPlanner,modules.aichat.datamodelFeatureAiChat,function generateTaskPlan,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.datamodels.datamodelAi,header,Yes +gateway.modules.workflows.processing.core.taskPlanner,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.promptGenerationTaskplan,header,Yes gateway.modules.workflows.processing.core.taskPlanner,modules.workflows.processing.shared.stateTools,header,Yes gateway.modules.workflows.processing.core.taskPlanner,typing,header,Yes @@ -2223,8 +2186,8 @@ gateway.modules.workflows.processing.core.validator,typing,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,datetime,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeAutomation,json,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,logging,header,Yes -gateway.modules.workflows.processing.modes.modeAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeAutomation,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.modes.modeAutomation,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.shared.timeUtils,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.modes.modeBase,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,modules.workflows.processing.shared.stateTools,header,Yes @@ -2233,8 +2196,8 @@ gateway.modules.workflows.processing.modes.modeAutomation,uuid,header,Yes gateway.modules.workflows.processing.modes.modeAutomation,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeBase,abc,header,Yes gateway.modules.workflows.processing.modes.modeBase,logging,header,Yes -gateway.modules.workflows.processing.modes.modeBase,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeBase,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.modes.modeBase,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.actionExecutor,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.messageCreator,header,Yes gateway.modules.workflows.processing.modes.modeBase,modules.workflows.processing.core.taskPlanner,header,Yes @@ -2243,13 +2206,11 @@ gateway.modules.workflows.processing.modes.modeBase,typing,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,datetime,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,json,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,logging,header,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,function _refineDecide,Yes -gateway.modules.workflows.processing.modes.modeDynamic,modules.aichat.datamodelFeatureAiChat,function _refineDecide,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _actExecute,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelAi,function _refineDecide,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _actExecute,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _planSelect,Yes gateway.modules.workflows.processing.modes.modeDynamic,modules.datamodels.datamodelDocref,function _planSelect,Yes @@ -2274,7 +2235,7 @@ gateway.modules.workflows.processing.modes.modeDynamic,typing,header,Yes gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.modes.modeDynamic,uuid,function _createActionItem,Yes gateway.modules.workflows.processing.shared.executionState,logging,header,Yes -gateway.modules.workflows.processing.shared.executionState,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.executionState,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.shared.executionState,typing,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,importlib,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,inspect,header,Yes @@ -2284,42 +2245,35 @@ gateway.modules.workflows.processing.shared.methodDiscovery,pkgutil,header,Yes gateway.modules.workflows.processing.shared.methodDiscovery,typing,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,json,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,logging,header,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.datamodelFeatureAiChat,function extractReviewContent,Yes -gateway.modules.workflows.processing.shared.placeholderFactory,modules.aichat.interfaceFeatureAiChat,function extractLatestRefinementFeedback,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.datamodels.datamodelChat,function extractReviewContent,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.datamodels.datamodelChat,function extractReviewContent,Yes gateway.modules.workflows.processing.shared.placeholderFactory,modules.interfaces.interfaceDbApp,function extractLatestRefinementFeedback,Yes +gateway.modules.workflows.processing.shared.placeholderFactory,modules.interfaces.interfaceDbChat,function extractLatestRefinementFeedback,Yes gateway.modules.workflows.processing.shared.placeholderFactory,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.shared.placeholderFactory,typing,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,json,header,Yes -gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.methodDiscovery,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,modules.workflows.processing.shared.placeholderFactory,header,Yes gateway.modules.workflows.processing.shared.promptGenerationActionsDynamic,typing,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,logging,header,Yes -gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,modules.workflows.processing.shared.placeholderFactory,header,Yes gateway.modules.workflows.processing.shared.promptGenerationTaskplan,typing,header,Yes gateway.modules.workflows.processing.shared.stateTools,logging,header,Yes gateway.modules.workflows.processing.shared.stateTools,typing,header,Yes gateway.modules.workflows.processing.workflowProcessor,json,header,Yes gateway.modules.workflows.processing.workflowProcessor,logging,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,function fastPathExecute,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.aichat.datamodelFeatureAiChat,function persistTaskResult,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelAi,function fastPathExecute,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.datamodels.datamodelWorkflow,function initialUnderstanding,Yes gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.shared.jsonUtils,function initialUnderstanding,Yes gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeAutomation,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeBase,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.modes.modeDynamic,header,Yes gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.shared.stateTools,header,Yes -gateway.modules.workflows.processing.workflowProcessor,modules.workflows.processing.shared.stateTools,function persistTaskResult,Yes gateway.modules.workflows.processing.workflowProcessor,time,function generateTaskPlan,Yes gateway.modules.workflows.processing.workflowProcessor,time,function executeTask,Yes gateway.modules.workflows.processing.workflowProcessor,traceback,function fastPathExecute,Yes @@ -2327,11 +2281,8 @@ gateway.modules.workflows.processing.workflowProcessor,typing,header,Yes gateway.modules.workflows.workflowManager,asyncio,header,Yes gateway.modules.workflows.workflowManager,json,header,Yes gateway.modules.workflows.workflowManager,logging,header,Yes -gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _executeTasks,Yes -gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes -gateway.modules.workflows.workflowManager,modules.aichat.datamodelFeatureAiChat,function _sendFirstMessage,Yes +gateway.modules.workflows.workflowManager,modules.datamodels.datamodelChat,header,Yes +gateway.modules.workflows.workflowManager,modules.datamodels.datamodelChat,header,Yes gateway.modules.workflows.workflowManager,modules.datamodels.datamodelWorkflow,function _executeTasks,Yes gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.methodDiscovery,function workflowStart,Yes @@ -2340,6 +2291,10 @@ gateway.modules.workflows.workflowManager,modules.workflows.processing.shared.st gateway.modules.workflows.workflowManager,modules.workflows.processing.workflowProcessor,header,Yes gateway.modules.workflows.workflowManager,typing,header,Yes gateway.modules.workflows.workflowManager,uuid,header,Yes +gateway.scripts.script_analyze_function_imports,collections,header,Yes +gateway.scripts.script_analyze_function_imports,csv,header,Yes +gateway.scripts.script_analyze_function_imports,pathlib,header,Yes +gateway.scripts.script_analyze_function_imports,typing,header,Yes gateway.scripts.script_analyze_imports,ast,header,Yes gateway.scripts.script_analyze_imports,csv,header,Yes gateway.scripts.script_analyze_imports,os,header,Yes @@ -2376,6 +2331,24 @@ gateway.scripts.script_db_export_migration,psycopg2,header,Yes gateway.scripts.script_db_export_migration,psycopg2.extras,header,Yes gateway.scripts.script_db_export_migration,sys,header,Yes gateway.scripts.script_db_export_migration,typing,header,Yes +gateway.scripts.script_generate_container_diagram,collections,header,Yes +gateway.scripts.script_generate_container_diagram,csv,header,Yes +gateway.scripts.script_generate_container_diagram,html,header,Yes +gateway.scripts.script_generate_container_diagram,math,header,Yes +gateway.scripts.script_generate_container_diagram,pathlib,header,Yes +gateway.scripts.script_generate_container_diagram,typing,header,Yes +gateway.scripts.script_generate_import_diagram,collections,header,Yes +gateway.scripts.script_generate_import_diagram,csv,header,Yes +gateway.scripts.script_generate_import_diagram,html,header,Yes +gateway.scripts.script_generate_import_diagram,pathlib,header,Yes +gateway.scripts.script_generate_import_diagram,typing,header,Yes +gateway.scripts.script_generate_import_diagram,xml.etree.ElementTree,header,Yes +gateway.scripts.script_remove_redundant_imports,ast,header,Yes +gateway.scripts.script_remove_redundant_imports,collections,header,Yes +gateway.scripts.script_remove_redundant_imports,csv,header,Yes +gateway.scripts.script_remove_redundant_imports,pathlib,header,Yes +gateway.scripts.script_remove_redundant_imports,re,header,Yes +gateway.scripts.script_remove_redundant_imports,typing,header,Yes gateway.scripts.script_security_encrypt_all_env_files,argparse,header,Yes gateway.scripts.script_security_encrypt_all_env_files,datetime,header,Yes gateway.scripts.script_security_encrypt_all_env_files,modules.shared.configuration,header,Yes @@ -2418,13 +2391,13 @@ gateway.tests.conftest,pathlib,header,Yes gateway.tests.conftest,sys,header,Yes gateway.tests.functional.test01_ai_model_selection,asyncio,header,Yes gateway.tests.functional.test01_ai_model_selection,base64,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.aichat.aicore.aicoreModelRegistry,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.aichat.aicore.aicoreModelSelector,header,Yes -gateway.tests.functional.test01_ai_model_selection,modules.aichat.serviceAi.mainServiceAi,function initialize,Yes +gateway.tests.functional.test01_ai_model_selection,modules.aicore.aicoreModelRegistry,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.aicore.aicoreModelSelector,header,Yes gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test01_ai_model_selection,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test01_ai_model_selection,modules.interfaces.interfaceAiObjects,function initialize,Yes gateway.tests.functional.test01_ai_model_selection,modules.services,header,Yes +gateway.tests.functional.test01_ai_model_selection,modules.services.serviceAi.mainServiceAi,function initialize,Yes gateway.tests.functional.test01_ai_model_selection,os,header,Yes gateway.tests.functional.test01_ai_model_selection,sys,header,Yes gateway.tests.functional.test02_ai_models,asyncio,header,Yes @@ -2438,21 +2411,21 @@ gateway.tests.functional.test02_ai_models,json,header,Yes gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,json,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,logging,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function testModel,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicoreModelRegistry,function getAllAvailableModels,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicorePluginPerplexity,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.aicore.aicorePluginTavily,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.datamodelFeatureAiChat,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.serviceAi.mainServiceAi,function initialize,Yes -gateway.tests.functional.test02_ai_models,modules.aichat.serviceExtraction.mainServiceExtraction,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aicore.aicoreModelRegistry,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aicore.aicoreModelRegistry,function testModel,Yes +gateway.tests.functional.test02_ai_models,modules.aicore.aicoreModelRegistry,function getAllAvailableModels,Yes +gateway.tests.functional.test02_ai_models,modules.aicore.aicorePluginPerplexity,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.aicore.aicorePluginTavily,function initialize,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,header,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function _getTestPromptForOperation,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function getAllAvailableModels,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelAi,function testModelOperation,Yes +gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelChat,function initialize,Yes gateway.tests.functional.test02_ai_models,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test02_ai_models,modules.services,header,Yes +gateway.tests.functional.test02_ai_models,modules.services.serviceAi.mainServiceAi,function initialize,Yes +gateway.tests.functional.test02_ai_models,modules.services.serviceExtraction.mainServiceExtraction,function initialize,Yes gateway.tests.functional.test02_ai_models,modules.shared.configuration,function _testTavilyDirect,Yes gateway.tests.functional.test02_ai_models,os,header,Yes gateway.tests.functional.test02_ai_models,sys,header,Yes @@ -2464,15 +2437,15 @@ gateway.tests.functional.test03_ai_operations,datetime,header,Yes gateway.tests.functional.test03_ai_operations,json,function printSummary,Yes gateway.tests.functional.test03_ai_operations,json,function testOperation,Yes gateway.tests.functional.test03_ai_operations,logging,function initialize,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.datamodelFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function initialize,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function _prepareTestImageDocument,Yes -gateway.tests.functional.test03_ai_operations,modules.aichat.interfaceFeatureAiChat,function testOperation,Yes gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelChat,header,Yes +gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelChat,function _prepareTestImageDocument,Yes gateway.tests.functional.test03_ai_operations,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbChat,function initialize,Yes +gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbChat,function _prepareTestImageDocument,Yes +gateway.tests.functional.test03_ai_operations,modules.interfaces.interfaceDbChat,function testOperation,Yes gateway.tests.functional.test03_ai_operations,modules.services,function initialize,Yes gateway.tests.functional.test03_ai_operations,modules.workflows.methods.methodAi,function initialize,Yes gateway.tests.functional.test03_ai_operations,os,header,Yes @@ -2488,12 +2461,12 @@ gateway.tests.functional.test04_ai_behavior,asyncio,header,Yes gateway.tests.functional.test04_ai_behavior,glob,function _getLatestDebugResponse,Yes gateway.tests.functional.test04_ai_behavior,json,header,Yes gateway.tests.functional.test04_ai_behavior,logging,function initialize,Yes -gateway.tests.functional.test04_ai_behavior,modules.aichat.datamodelFeatureAiChat,function initialize,Yes -gateway.tests.functional.test04_ai_behavior,modules.aichat.interfaceFeatureAiChat,function initialize,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelChat,function initialize,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test04_ai_behavior,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.functional.test04_ai_behavior,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test04_ai_behavior,modules.interfaces.interfaceDbChat,function initialize,Yes gateway.tests.functional.test04_ai_behavior,modules.services,header,Yes gateway.tests.functional.test04_ai_behavior,os,header,Yes gateway.tests.functional.test04_ai_behavior,sys,header,Yes @@ -2504,10 +2477,10 @@ gateway.tests.functional.test04_ai_behavior,uuid,function initialize,Yes gateway.tests.functional.test05_workflow_with_documents,asyncio,header,Yes gateway.tests.functional.test05_workflow_with_documents,json,header,Yes gateway.tests.functional.test05_workflow_with_documents,logging,function initialize,Yes -gateway.tests.functional.test05_workflow_with_documents,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test05_workflow_with_documents,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.datamodels.datamodelChat,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test05_workflow_with_documents,modules.interfaces.interfaceDbChat,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.services,header,Yes gateway.tests.functional.test05_workflow_with_documents,modules.workflows.automation,header,Yes gateway.tests.functional.test05_workflow_with_documents,os,header,Yes @@ -2518,10 +2491,10 @@ gateway.tests.functional.test05_workflow_with_documents,typing,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,asyncio,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,json,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,logging,function initialize,Yes -gateway.tests.functional.test06_workflow_prompt_variations,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test06_workflow_prompt_variations,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.datamodels.datamodelChat,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test06_workflow_prompt_variations,modules.interfaces.interfaceDbChat,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.services,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,modules.workflows.automation,header,Yes gateway.tests.functional.test06_workflow_prompt_variations,os,header,Yes @@ -2533,13 +2506,13 @@ gateway.tests.functional.test06_workflow_prompt_variations,traceback,function te gateway.tests.functional.test06_workflow_prompt_variations,traceback,function runAllTests,Yes gateway.tests.functional.test06_workflow_prompt_variations,typing,header,Yes gateway.tests.functional.test07_json_merge,json,header,Yes -gateway.tests.functional.test07_json_merge,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test07_json_merge,modules.services.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test07_json_merge,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test07_json_merge,os,header,Yes gateway.tests.functional.test07_json_merge,sys,header,Yes gateway.tests.functional.test07_json_merge,traceback,header,Yes gateway.tests.functional.test08_json_finalization,json,header,Yes -gateway.tests.functional.test08_json_finalization,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test08_json_finalization,modules.services.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test08_json_finalization,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test08_json_finalization,os,header,Yes gateway.tests.functional.test08_json_finalization,sys,header,Yes @@ -2549,10 +2522,10 @@ gateway.tests.functional.test09_document_generation_formats,asyncio,header,Yes gateway.tests.functional.test09_document_generation_formats,base64,header,Yes gateway.tests.functional.test09_document_generation_formats,json,header,Yes gateway.tests.functional.test09_document_generation_formats,logging,function initialize,Yes -gateway.tests.functional.test09_document_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test09_document_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test09_document_generation_formats,modules.datamodels.datamodelChat,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test09_document_generation_formats,modules.interfaces.interfaceDbChat,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.services,header,Yes gateway.tests.functional.test09_document_generation_formats,modules.shared.configuration,function initialize,Yes gateway.tests.functional.test09_document_generation_formats,modules.workflows.automation,header,Yes @@ -2568,10 +2541,10 @@ gateway.tests.functional.test10_document_generation_formats,asyncio,header,Yes gateway.tests.functional.test10_document_generation_formats,base64,header,Yes gateway.tests.functional.test10_document_generation_formats,json,header,Yes gateway.tests.functional.test10_document_generation_formats,logging,function initialize,Yes -gateway.tests.functional.test10_document_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test10_document_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test10_document_generation_formats,modules.datamodels.datamodelChat,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test10_document_generation_formats,modules.interfaces.interfaceDbChat,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.services,header,Yes gateway.tests.functional.test10_document_generation_formats,modules.shared.configuration,function initialize,Yes gateway.tests.functional.test10_document_generation_formats,modules.workflows.automation,header,Yes @@ -2587,10 +2560,10 @@ gateway.tests.functional.test11_code_generation_formats,csv,header,Yes gateway.tests.functional.test11_code_generation_formats,io,header,Yes gateway.tests.functional.test11_code_generation_formats,json,header,Yes gateway.tests.functional.test11_code_generation_formats,logging,function initialize,Yes -gateway.tests.functional.test11_code_generation_formats,modules.aichat.datamodelFeatureAiChat,header,Yes -gateway.tests.functional.test11_code_generation_formats,modules.aichat.interfaceFeatureAiChat,header,Yes +gateway.tests.functional.test11_code_generation_formats,modules.datamodels.datamodelChat,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.datamodels.datamodelUam,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.interfaces.interfaceDbApp,function __init__,Yes +gateway.tests.functional.test11_code_generation_formats,modules.interfaces.interfaceDbChat,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.services,header,Yes gateway.tests.functional.test11_code_generation_formats,modules.shared.configuration,function initialize,Yes gateway.tests.functional.test11_code_generation_formats,modules.workflows.automation,header,Yes @@ -2603,7 +2576,7 @@ gateway.tests.functional.test11_code_generation_formats,typing,header,Yes gateway.tests.functional.test11_code_generation_formats,xml.etree.ElementTree,header,Yes gateway.tests.functional.test12_json_split_merge,asyncio,header,Yes gateway.tests.functional.test12_json_split_merge,json,header,Yes -gateway.tests.functional.test12_json_split_merge,modules.aichat.serviceAi.subJsonMerger,header,Yes +gateway.tests.functional.test12_json_split_merge,modules.services.serviceAi.subJsonMerger,header,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonContinuation,header,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function _loadTableJsonExample,Yes gateway.tests.functional.test12_json_split_merge,modules.shared.jsonUtils,function testJsonSplitMerge,Yes @@ -2632,22 +2605,22 @@ gateway.tests.functional.test14_json_continuation_context,traceback,function tes gateway.tests.functional.test14_json_continuation_context,traceback,function runTest,Yes gateway.tests.functional.test14_json_continuation_context,typing,header,Yes gateway.tests.functional.test_kpi_full,json,header,Yes -gateway.tests.functional.test_kpi_full,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_full,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test_kpi_full,modules.services.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_full,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test_kpi_full,os,header,Yes gateway.tests.functional.test_kpi_full,pytest,header,Yes gateway.tests.functional.test_kpi_full,sys,header,Yes gateway.tests.functional.test_kpi_incomplete,json,header,Yes -gateway.tests.functional.test_kpi_incomplete,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_incomplete,modules.datamodels.datamodelAi,header,Yes +gateway.tests.functional.test_kpi_incomplete,modules.services.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_incomplete,modules.shared.jsonUtils,header,Yes gateway.tests.functional.test_kpi_incomplete,os,header,Yes gateway.tests.functional.test_kpi_incomplete,pytest,header,Yes gateway.tests.functional.test_kpi_incomplete,sys,header,Yes gateway.tests.functional.test_kpi_incomplete,traceback,header,Yes gateway.tests.functional.test_kpi_path,json,header,Yes -gateway.tests.functional.test_kpi_path,modules.aichat.serviceAi.subJsonResponseHandling,header,Yes +gateway.tests.functional.test_kpi_path,modules.services.serviceAi.subJsonResponseHandling,header,Yes gateway.tests.functional.test_kpi_path,os,header,Yes gateway.tests.functional.test_kpi_path,sys,header,Yes gateway.tests.functional.test_kpi_path,traceback,header,Yes @@ -2656,7 +2629,7 @@ gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUa gateway.tests.integration.rbac.test_rbac_database,modules.datamodels.datamodelUam,function testBuildRbacWhereClauseUserConnectionTable,Yes gateway.tests.integration.rbac.test_rbac_database,modules.shared.configuration,header,Yes gateway.tests.integration.rbac.test_rbac_database,pytest,header,Yes -gateway.tests.integration.workflows.test_workflow_execution,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelChat,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelDocref,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.integration.workflows.test_workflow_execution,modules.datamodels.datamodelWorkflow,function test_extractContentParameters_structure,Yes @@ -2687,8 +2660,8 @@ gateway.tests.unit.rbac.test_rbac_permissions,modules.security.rbac,header,Yes gateway.tests.unit.rbac.test_rbac_permissions,pytest,header,Yes gateway.tests.unit.rbac.test_rbac_permissions,unittest.mock,header,Yes gateway.tests.unit.services.test_json_extraction_merging,json,header,Yes -gateway.tests.unit.services.test_json_extraction_merging,modules.aichat.serviceExtraction.mainServiceExtraction,header,Yes gateway.tests.unit.services.test_json_extraction_merging,modules.datamodels.datamodelExtraction,header,Yes +gateway.tests.unit.services.test_json_extraction_merging,modules.services.serviceExtraction.mainServiceExtraction,header,Yes gateway.tests.unit.services.test_json_extraction_merging,os,header,Yes gateway.tests.unit.services.test_json_extraction_merging,sys,header,Yes gateway.tests.unit.services.test_json_extraction_merging,traceback,function main,Yes @@ -2696,11 +2669,11 @@ gateway.tests.unit.utils.test_json_utils,json,header,Yes gateway.tests.unit.utils.test_json_utils,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.unit.utils.test_json_utils,modules.shared.jsonUtils,header,Yes gateway.tests.unit.utils.test_json_utils,pytest,header,Yes -gateway.tests.unit.workflows.test_state_management,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.unit.workflows.test_state_management,modules.datamodels.datamodelChat,header,Yes gateway.tests.unit.workflows.test_state_management,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.unit.workflows.test_state_management,pytest,header,Yes gateway.tests.unit.workflows.test_state_management,uuid,header,Yes -gateway.tests.validation.test_architecture_validation,modules.aichat.datamodelFeatureAiChat,header,Yes +gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelChat,header,Yes gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelDocref,header,Yes gateway.tests.validation.test_architecture_validation,modules.datamodels.datamodelWorkflow,header,Yes gateway.tests.validation.test_architecture_validation,modules.shared.jsonUtils,header,Yes diff --git a/scripts/import_diagram.drawio b/scripts/import_diagram.drawio new file mode 100644 index 00000000..64e6a59b --- /dev/null +++ b/scripts/import_diagram.drawio @@ -0,0 +1 @@ +7V3bduM2sv2afhwv8U49+tKd9Kz2iVe7czJvXjQFy5ymSA1Jte18/QElQReDpWgSHWykhJmsxLpYpjY2gdq7CoUPwfXs9acmmz/f1hNRfvBHk9cPwc0H34/D2JP/6Z95Wz3jhYG/embaFJP1c9sn7ovfxfrJ0frZRTER7d4bu7ouu2K+/2ReV5XIu73nsqapX/bf9lSX+391nk2F9sR9npX6s78Vk+559WwwGo22L/wsiunz+k/7m1dmmXr3+on2OZvULztPBR8/BNdNXXern2av16Ls4VPArH7vE/Hq5soaUXXH/IKEqMuKSjQPWZE/Z93q939k5WL9XXefbrs3BUH7UszKrJKPrp7kR9yvX/Hk4/y5KCdfsrd60V9B22X5d/Xo6rluit/7P1mu3yxfbrr1CAejvXfc978pn+6fbUQr33Onvpb37qnb7HXvjV+ytls/kddlmc3b4nFzfbOsmRbVVd119Wz9ppfnohP38yzv3/MiWdtfSDdTF/lUlOV1XdbN8rsHk0ikk1A+r4O9xv+HaDrxuvPUGvyfRD0TXfMm37J+9R9RsGbC+mYIxuvHL3vMWj33vEsqT/1mtqbzdPPp2wGXP6zHfHj8K3l3PvhDo1434mL1n6usFRoB5PfrluPX1N+FAqeqV4zYwWv9VFYW00o+LMVT/2s9QIW8oS7XT8+KyaT/5KtWDkJRTb8s33YTbp/5uv7m/VO1/PWncnnbPMtfFPITruZ1UXVLIKIr+Y+E5np0EX2I5LVey8fe9rH8p397013Xlbx8yf7+c4WkzIvoaXPV1F3WZSvCHMOOnv8rBnujQVYQN9kfk0Rx4khKnIwQwWFCLKf0r2JatOrDHTMwzIhNMyM8ghn3opSrrhxjxwwcM45dRk7GjOgwM+7KhVx1L6vuuannRe64AeSG55smR3wMOT5XnWgqGRo6bgC5EZnmRnIMN36ZiyorHDOQzEhNMyM9hhl3opmX4rXoXCSKZMfGnDHGjvEx7PiW/ShKxwwoM0LTzFBU1Gysi4mEZtZrlE8i6xaNuCyuh9wtRw+D9EiM08Oj6FH0EeiTxMLRwxZ6bKhgjh6DdmhPj6ZedI4a9lDDuC/q6cZoK5ofRS7pcDGT3+VePXLEQBLDuC3q6b7olhjt4lHOFllZfqnruRwaxw0kN4wbo57ujO5xQ0LXya/38VXil3dFXTl+APkRGvdGPd0c3ePHTZ0vZvL7fV7SpHXsQLLDuDvq6fboHjv+2dbVrWimwqXboMQwbo56ujuqEeOraOcSQfFzVk1KF3lgKRIZd0g93SLdo4iix13WtI4dYHYYd0lVoEOx475rFnlvdXySQ+3oAaaHcZfU113SYXr8JOTXypxwgZeCGWeIbpTmz4vquxyIi+UPovk860upHS2AtDBukvq6SfqeFpu5w1EDSQ3jNqmv26TvqfFtiZCjBZAWxh1SX3dINVr0BHCswLEiMe6L+rovKlYeed20F5sfr4oqc7sQsNww7or6uis6xI3r9ocjBpIYxl1RX3dFh4hxU+evjhlAZqTGzVBfN0OHmPFzj4JjBpAZxo1Q9UF/wAznZcCpYdwEDXQTdIgafa7NMQO629E4M3Tzc4gZd5MnRwwkMczvndftz0FizDsXgUKZYdz9DHT3c4gZ9/9xASiUGMb9z0D3P4eI4TxQ9A7YkXETNDjOBP1X2brVBEsN4x5ocJwH+i/nZ4CZYdwEDcjS0O0mg93tSm7rgR1M8YybooFuis5E3x5vejFbFpXfiKdsUbqwA8sL45ao+oMkL1yZBp4Vxt3QUHdD37HCKRR4Zy/jpNCNUD3YaBePbpOSBeww7oaG5I75fXbcFXNRyq/m+AHlh/l+ouTG+Xf8aOrZvLtaFOVENE6vWEIX405pSO6l36eLa05sAz8C43ZpSO6l3+5S2nU+3N4lS5hi3D0Ndfd0nnXP7UUuX73rj2lwhEASwrhpGuqm6YoQk3X/DUcKNClC4/5oqPujK1IUfTGgYwScEcadUbVQ7TBCfh8pSkSzWju+rh/1p6Z8E7N5mXVO1WJJYtwojXSjdEsStZ44othGlMi4eRrp5umWKI0TsTaQwrhnGume6S4pVj9dy3e67W1obhj3SyPdLx3mhtuSACeH+SOYdHd0mByuJgzMjdi4MxrpzugAN9yCAuaFcR800n1QnRduuzScGMb90Ej3Q3ViuN3SaGIkxj3RSPdEdWK4zdJ4Zhj3RpUaOsgMJ0zgxDDuh8aH/FD1023WfJ/UL44cUHKkxj3Q+LAHuvrJ7aNH88K4DRofY4O6bfRwYhj3QONjPFC3FQFODOP+Z3yM/+m2SqOJMTZ//PwxZaHbo7jWT8oRdzxB8sS4GRqThy0N8qQ/kWvqiIIninFzNCZ32O8TRZ3d9mtXlO6EevBB0yPjVmlMHsC0T5O93U1uT4IldDHunypf7o/o8v7oHbf+gJli3FBNyMOZfhOPuxuc5EPHDSQ3PJN+6s7VzucaQzbP7XChfSlmZbYa+h6M9Ss9OPmzXI6+ZG/1or/Ctsvy7+rR1XPdFL/3f0whKV9uFJTBaO8d9/1vrsejEa18z50C2Hv31G32uvfGL1k/qssn8ross3lbPG6ub5b1rSeu6q6rZ0cO9w65P/jBJBPpUz447N7R47x+9egNbBs+/PU5QPfNh0bY3e1/6m7f3D/HnyyOudEXag/SLg8WAxuTzvlWf3oScX6iW33bRUMN/LGBou+d7N7XcyODY+5u/j9182/vKSvv/hUF9CxIf90X/b/kd5Lj5iQklBHG82KJnv5YMuLfL91aDTg2wNhgPBmW6DmPJRs6OdLVbVZlrhsXkg+e8SRYoic3toT4Kp5kHPZ8uxyxF/ktHTVw1DCe90r0dIZGDbeEwHlhMs21vVT5UyWWTcU1krx/yanNpdoU3iQSyWnU5j+2vZE2SuNYu9lPTjZB6Ims7dhfbH68ebyr227qVo8TzRLarWevHFVz0x8w5Fa0bdZ3DP44ywq3dQhNE+MaNdUTVwdpcj/TlxZHErMkMS5dUz2zMUSSu0bMmzqXVHEJcDhJzOvZVPfAh1hy/1K07bd6Xt9m817DONcDzxXjAjfVzfIhrnwr8u+ia69L+d+FS6XCeWK8rjPVLfQDPPln0WSOJGCS+MarOlPdWR8iyf/WfeFeXU9LJ4fhJDFZy7m94IlEYiYpU+rW2fuXnHW2tM6enp78UxVqvDfO/PGxU8U4PNlcoSddDoy8mxb+zLSg3WUWu2R6omV79RebHy8LRw0wNYyXdKe6xT5IjcWkcBub0ewwf1KnMuH+gB43dd4I1yoDzg/jDsdYN9YJfiw3JDqGoBli3FUf6676EEPc4WrWcMR8fDrWPfUhjnwSWb/90AkYOEOMh6lj3UkfZEhROnrA6RGYD1N1A32IHp+rH0XnNihYwRHzoarunw9xxDUKtYEd5sPUQ47plh23YvYomva5cGlaNEdC82HqcTbqporMUQRNEfNx6nF26l3PDxeI2MCRyHiwuunF8Ack+fqY5Y4eaHoYj1O90XGe6r3IF43r7mYBRYwHq97oOFN1XTDmGAJmSGw8VPVGx1mq3+raFYTg+WE8TvVGxxmqv2Yzxw50TQggQj3OT+37y7rZA84PQIh6nJe6rEV2/EDzAxCfHuem/lY335ej4SiCpYj5Y3a80XFmqqLI5bI2xC02cKaYDFW3F/y0Lv3oO5jUsx3ndIc+6j0X79/jNjn03z0QvniaZIPM+K83OSRqyP/rTQ6n60TrjXSzfTv0F40cOrEuGbqkGeEmkD8zgdD3o5W7IAYuvG+r/Vh3GoM2s8jeG9wUcropZJvjP5oBJ5sy1Cy1u4lyNc5qvJc3U1Y51/TUM8X+DWflNLHmyECzmDVH3peiXhNzhKMKgirmXVRPz8Moqogfy9nZNTy1hx/mt0953kC3mDVBJHJyICQubi6xkSvmTVVvoFvMmiv90TqOH1bxw7yp6g10iVnzY1frOp7YxBO0zF3/8FVMi1Z92qDcHXyjk70nd860vrpHy+ATOmeenr87mghuHvlL88jwDWmlHB64+kosJHCl/NINzSDtTW4a+X834I+fRjbdd08wjehp3p2x19yS/znACzer/KVZRb8vrZxR1rTRU7+7tHkvjB1tbKQNwGzT0327tOk18pYqd2X2NpWyqJo4zljDGYABp/pDE6TZFc5unrGSM+aNOF9PAK2Pud5QZFVisHPi9f4rjj/28MekUTdw6Y3ISglU1glaML1/j9NLp9NLaiuwf+w84vunm0j09OB2qDV59FW+9HGYBW76+EvTh3YPWiyOfD1juMOZ99rIccY+zphXRr6eOdzhTB+lOJ7YxxOEGtJTiDtE2RVDjjAWEsakFBq48q5ZtJ04EMfuvcEFsSc3/d/nDo8Pajdn2pxgDtFzh+tx1yLabwQf3Azyl2aQ/fvQ5lhWzw8pqrwPZB1VbKIKIITVc0KKKrthiaOJTTQxGsFuL3szd+jnG71/yQUhyyDkKc3Fqc43el95cHwQEp7QWdNzgduh3y4ul8Uvj/8WudvMc6IJQ7vzLA4+1Cf9AUWu6rrrYZw7ioApYj7oCPRE3xBFbh4v544eaHoA7LJAz98M82O1z8udcGMDTczXCwR6ymaIJu74ElsoYn7vTqBnaIYo4lqH28IRRMCqJ2eGOOLaQttAD0CwqvvuQ/RY9fx1stcOnpg/CskLdNN9iCfL3oyOJpbQBJPZXTrruom6+7QzUJcG6mSUC+EPjv9/X4qY+EeO9ukOhPcCPb+yGuZVeuVyMitc1fJppoG9u8pmg1T30DVKbBvnfez72biVAkkR8zGnmn8OUcTZGhZQA2COhrp5rnGj16ofX3skHTuQ7DDviYa6dT7Ijq+1O7EXTA7zbmg4sMdhkBwLRw4sOQBhaTiwmWGXHF3XFI+DAtURwyAxAMHowOaFHWLcSIAkfJVwJwXA2QGwPUPdHn/Pjk+FW07AvAAEorod/p4Xt1k1ydyCAqYGIAw97In21Lhr6tnc2V5QZpg/6N0LD1ujPTN+bUXjeAHlhfkQVC1fB3ihDqxy3EByA3Cqe3TYD/1pMm8cJZCUMB97RodN0M/Vj6LLnFqFM8N86BkddkBdHagNvAAc2x4dNj/vRb5oiu7NlWvAuQEIPg/7n4obP9X1tHQ7pKGFGoDo87D9qcjxpZaD6LiB5AYgDD1sgSpu3LZPrkwDSg1AHHrYAr1/ll9vCbkjBpAYgOPZo8MO6HL3gIs08MzAHMferpcMjSP7L7jNA/13D56ySZKdqPuK1gLu6M0EJ+xrrE6QGRj4i3nWti91M/m1K0pna51mbnh3u1m8pyAeap2/ZkbjNq6iCWHetIh1C3yPED+Lcu4SqWheALYPxLoDviVGXXeXeS5axwswLzD7Utenreg7U/dfcOHlMrx89CaTp2EG/PXmfkeHlyds7hfrSRBy4N188Ofmg70bzOaAUk96qIu/eHgoqqJ7eHCkgJICEFTqyY71xfeHn+8ey9U/dvSA0gMRW+oJj/XVbwordkniqi0sYYr51Fis5z/axWObN8V8WZd1IR/d7zxx/9Z2YvaxaWqnWdFsMZ8ti/WkyPrqVQ51d1q5p7xwxxOTPAEErSoiGuDJJqG6xxSXZrWEK+Zj2YQ8bHbVyW+XJ6tnHEewGth8PJsMmejLq19m2XYp4tJuFjAE5Jr2q8hEZ8rO084xXTqmk1BMTnWwcHR0sWd0ujPYkoHkyfAwuzv/T935u7eSxe5oMuCULy/9IlNtUtyCgKYFIKYcMM3XtFhMiu5LPZ2KxnECyAmAKZoMmOYrUshRKx/lUv1VTItWfbZjBogZ5k3QZMAuXzOjrp6K6aJZblJ1tEDSwrzbmQx44ytaTB5vF2VXfBNVVnW/zLtiVvzu9jHDOYIIQAcc8TVHxONi6iINPCnMh59K5+qkED+WWt8dlmYBMQA+Zjrgda+I8dTIbyOqybe3uSvywtLCfPyZDtjbK1r8u60riVxXVAsXgsKZYT4ETSmfc8kMZ22hKQFozZeSlue8qaeNaFsXdOJ5AQg6Sc9TylPngsMpYbQj3/ZiX/bbMe6w44Vs1HjOqdGnqP//IAP+8l7l41OlXjo+3cSg+96bob/INudbucnhJJPD+/vN4uRpqtveWzosa2pUL1fHDSw3YvPc0L3vHW60i8ftwXj3+bOYLFwfFDRJAOpUN7+z1eEjfeKsv/KbOl84oxPPDe/YXaynI4ei4wA5pkJ+mayT4z5xswaaGebtzrHugr9nhps3LGFHap4duhmu2DFvatcTA08K33zibKz74IoU7WImRb/8zm7OsIQeoXl66J64oodErWpLt6TYQw/z1vhYt8YVPV7E41fRiqzJnx0x0AaYeWLo1qj8pOd6cllcqB8cLcC0MG+MjnVjdMWGds2Kq6x1whXNC/Om6HjAFF2vI/9ZiOatP/7s0VEDTw3zVuhYt0JXU0XfgOmx7i72HjmCYAkSGrdDffXpA3OHHPtG/ny9rAB13EBzw7gh6o8OGaKbHMrnaiK/gWMHlh3GDVF/RBuilVhI4Mre/JJIOW5guWH+zFV/RPuiXVP0ZaB3jVjb5nKA+h4prjIUzhPjBqk/0g3SdUDaBx2vm/B09cgRBEwQ4xapP6ItUvmFKpF3/ywat8LA68DME0O3SDfEaERfxNH+cMrFDnYYd0r90UAJ6R47Pr7monT8sIMfxh1Tf0Q7puK1x2/V0bG9bP/ZuhJ0OEGM+6b+iC4hLWY7BPnU1DNHETxFjj4O5XQUUap6gCIz0UzXTWGdAWIBOcxbpx5tncov17ro1B5ymHdOPdo5XZLDBaf20CM1b556unn6vDqN8WKyzrncSZq4c1Hw5DDvmHqUY9r7YBfbHx01wNQw75V6h7xSKVpacVlNbprsqfs4y4ryt6J7dsa6FWQZm/dPPdo/bUQ2WTLE7WhB88K8c+rRzumqHN0xwwpmmPdMPdozbcXuwuK4AeaGebvUo8pMf1l0ZV1/v9h75AgC3j07Mu+Wqr84MHtkVVa+/S4+1eVENL+22dQVqsMJYt4x9WnHNK/nb58K18kFTwvzXqlPe6WT+qUq62zSU+Pq7S7r3FZJNEHMH58iP4okyFMhw9KNY+rogaeHeb/Up7fg9/S4l6hcvf3aONEC54Z5w9SnDdOyaDcbXJzZgeaGb94f9Q/7o44b1nDDvEfq0x7pYt5HpK6niy3kMG+T+rRNuiKHU7I2EMO8R+rrHqkr67CTHYF5f1R9kuag3/dd/ZeAX7x/wvEEzRPzNmmg26TbfbQX2SSbd8UPt77AmWHeKQ0GnNI1HTa8+CKyppLD87Gayi/oWAJmSWjeLg10uzSvG8mQZYz68VXki06OsWMGmBnmndJAd0qXzJjJtSWbiut+36RjBp4Z5n3SQPdJl8zosvb7XZlVlevdAadFZN4iDXSLdEkL+biYuLnCBlKY90aDgR6l8oX2ov/3pTvVyxpqmHdGA90Z3VLDta61gRTmXdFgoHJ0Q4qbtyqbFbnjBZgXsXk/VOkf/dhYsdSo/QFvEik3Z8C5Yd4DDXUPdM2NlUN+U7R5PxRvjhxocpi3QUPdBlXnkJcSjOdllfmnLJf6xPEDzY+jDx4+IT90A3R7Tv1s3v20OupNri+Xq7S9C0Is4Yp5SzTULVGCK9+y9rucX5yuhbPEvD0a6vbomiVtH6F+q2u3XxbOi9S8Pxrq/uhOql59lbvVU84txVPEvFsa6m7p5vo3DLnNqmzqUix4fpi0TMVk2vNDt0yfRTYZIENTL6qJmKzxktA+19O6ysovdT1fo/Rv0XVva5iyRVeTGKbyYZk9ivIqy79Pl5+suPTBD56W/+s/bzGb36+vIGvyQdzXSr2tF00udpifrFValzVT0e2+sA7P++9/cFQaUWbLUqfdv3MCxIeqNFkj7qlWMCjIlYF0RpCrKj4Y5LqDxh7yFAy57kuxh3wMhnyokSJvyJVHAoNcd0zYQ46ey3X7gT3k4CAx0pU9e8hDMOS6UmYPeQSG/OzEpxeipdD5qU9lgaEgj89PfYZgKRSfn/oMwVIoPj/1GYGlUHx+6jMCS6H4/NSnspVgkJ+f+ozA6jM+P/UZgdVnfH7qMwKrz/j81GcEVp/x+anPCKw+k/NTnxFYfSbnpz4jsPpUV7YD+VP1IbgsiyfRzgeqZtkgn4BX0eT8RGiCntLPT4SmYKslOT8RmoJjxYSrCKUQT9FhC1cJGhCAJ+ighasApQCHM5yr/KQA90ZgWyvlqj5JxFXqEYY4V/FJI+6DEde1Jw/EQ0un8ZSr5IxsBZyr4KQAh0v8lKvejG2lOFe1mdgKOFe1SQGOn1O4ys3UVopzlZsU4N4IXLgy5io3ScTRk8qYq9wcWzqpjLmqTQpw/KTCVW6SiMMnFV1vLrP5i/kk68Snpp7di1LkXXFxcfF3HwTVn26A9+hR4KpBacgTsF0+5qpCachTcP3KmKsO9TwCcnTac8xVh5KIj9GIcxWiJOLeCFswtOnadU6QYxfPYKRL0WXU+CCqpsift0eDcIgayVEAN34KRro+XY5C29WNuBGPi+nt6rgv3qPgg0eBq2Y9ADk2kAxGXJOkNOTgWpdgpCvU1Vwjul9b0VxLmPoOpnzxxzpjwUiXq6sVt6q74untTFbcBFsoHYzOTsGiDcpgpEvYMwxywEZCMDo/WQt2ywKPraz1CcjB7aIDxYXzQRy98yhQV8YPcks3ZgQeW7lKIa7qJmCIs1WrJOJgS8Zjm06ldx+Bw3SPbTqVhNwHx+QeWzFKQp6iI0S2+VRLNyAFHlvhSSGOXjx9trqTQtxL0ZCzFZ405OCZ3GcrPCnIfXAL+sDXlefSyRWvXZPl3Tf5377e9POMg5Nr6f7SwGcrRinE4espWzFKIY6fadiKUWpPL5zkbLUohTie5Gy1KLmtFx40shWjlu7rDQJdjC5jxpmQF/qpyaYz+d0+V13NIWakd56CncaArT4lIUdPNaqqlR/klu72DQK2iVFy8ymc5FzFqE/twYOTnKsYJRFHK6OAKtyVl3mZ52Leick9kw3W9CCAy4wCXZ4uB+FxUZQr9Ovqrqlnc86DAE9zBEQp78PaAb6sJrd9XH/LexTQiy5XEUtCDrdqQq4pVfWCdXFOyFWxkoij45xQF6zLyX0qKtFknbjvmkXeLRrBdgQ8dP16SCRYz2gI0CFOyFbRUpDjF1e2kta3darnml71bS1oD7mmV31bqyBDrtlV39Y6mZCtLrW1aCNiK0upCgI44mxlKZVIhSOuy1ImiFNZPTjiXPOoga0ppoir6Awo0QlHnKvmVGUn9iHOVXMGtmrOiKvmDGzVnBFXzRnYqjkjrpozsFVzxlw1Z2Cr5oy5as7A1qJ1dUrq+SAO5/jhraXL7qTV3747KQk/vAdyTJwicy74ozdoxGz1qLUbNGK2gtRWYzHmKkhDW43FmKsgVeWB9iHOVZCq8kDrEE+4CtLQVmMx4SpIQ1ubqCW6IN1uf/m4itGXO2AYbLwgBwEepCdcE6OhrfZuwjUxGpL2rgdGnKsQDSl7Fz63E/tLVdH/tbxItuDD6c5WktqazUjYSlKy6xG6tXeia1K1g307xfR7izhEj3TvKTDxU7Y61dp2XylboWpr5lTdZOeDODqCSdmKUltz1SlbUUrm6tARTMpWlVKQw3flplzTo2qbmn1rJ1stSvavg88rXMVoRGak0eEK1/yo2hlo3bQy5qo7ScTRHFd//4wQB7cHVJKAH+K2nok55io7I3JzERpxrrIzIrPPaMS5qs7I1uzzmKvojGxNgI6JxrrvmxtzOJ+BHgR0+MJWhtrqmo+JlChL2pODgKV9OCLOhtnkpS+Lz7Ns+revfoksdRzDEVuhSm6Bwc474YirUI0tNRzDEdlTl99MTw8Ceqbnql3V3nz7aK9rV760t9SWDEeHjo4pll+3/a3onu84DIJva5DDNa1KIg6fe4hjYljOPeQgoOceronW2NKUSOgRcpYl7clBANPe4yppY0vzUqGnS1q+tCcHAU17QuW+jzQ/chgEKl+IjjQ9tirX0gxt6J2TyiUHAT330Cp3fWxVMZuX4l5wGAQqcw6fe4g87lkNAnrDWehxTeSSkKP3KYQeW51L5rDA871P6lxmpxQmlh4CH6pPYsd5EnE453WRywRxS7cthD7XAuOE7G+l6mFgkHPVrwcgBwtYn2uJ8QHIQzDkXGuMacgDNMu5pmBpyGOwEvWJHGwj/rMoGnH/1l5OZkXFdwASsC71uerSA5Bjd3OHAdedrgll/eIhZ6tESchT8MSijgrhBzlp66Kj9ICtFqUhB0fpAXEgjbzMT41on7/V38XfP3oh4UdH7AGdWD0H+NHRe8BWo5KQwyMZthqVzpaiIxmu2dKE7oiKhpytKqV7F4IjmZBIlzbiqV9JH8TrvGjE5KFjUJlBjwI6oFEHRp33KKDjGnWI1BlNP2gfMmSrXemTJdGQs82j0o08wanrkG0elYYcHdew1agk5GiNGrLVqCTk8Lmcq0ZNqVI7PMu5alR1HIl9kEdcM6fqPBILIeeaOU2prcF4yLmqz5QutvPBkHNVnylZHwCHnKv6TK3NHUVc1WdKSyGw+oy4qk8acrQUiriqzzEthdAs56o+x3SQiIacq/oc03W7YMhjruqThhw9l8dc1eeYLpVGs5yr+qQhh7Ocq/ockwVGcJZzVZ9jOsOPhpyr+qQhh08sXNXnJpFvIc25ys/N9hYLMeeqPw9gDp9buArQTcWKfTxPuCrQA5ijea46zzDE3FoNmnDVoN6ITg6hMecqQg9gDp9buKpQb2St8k+4ytADmKP7jSZ8dSi9nwsMOV8Zaq3DlfCVobTFBd4wl/CVofSWCizkKV8VSkIOZrm6y84I8gQcs6iS7DOCXJ02DYOcOB1GvIp80YmHHrlZxuOYjANzTQwOY1K+qtTaOtGUryqlC0XBzdJTtqrUs/VEgJQ4/mUxn2Sd+NTUs3tR9ichcZjhVdA4EMCDNwKkbIUqjTl8hmcrVGnM0f76mK1S9ajC9AQcw6uj9s4IcrRsGrNVqiTk8EYuY7bZ0gOYg1fQMdFr90FUTZE/Xyp3oGIRPJLDgG5RNyZ67rZd3Ygb8biY3oq2zaYcjnA8NAw+eBj4qlcac3Q8yTapSmMeo6cb4oSYVnS/tqK5ljiJ147zAIATIWNduK6W3aruiqe3c1l2E6xbGY3OT8uiG2ZEqvr+zEMdsKcQqVY174dhIkrRie0EdMN7GFKs5o1GZ6h5wQ5mNGKbi/XoZmBgyNmmYknIVa0LDHK+WpaEHJuJjUZ8pSwF+Rg9sfDNutp6inI0osRrXlc/RNM9PBWleCgmLYvQkT5ZGRzGeHzFK320Mpj6ni5e2WOODtc9Xamyx3wMNsY8vrKU2qANro2PPL6qlIIcXOQReXxVKdmGAB618JWltvbCizy+upTGHL2C8hWmtrYhiDy+5cD04ZrgucXnq0JtbUEY+XxVKL0/Gzy3+HxVqK2bJiOf2Ls6b+pctO39s/x687qoOBzce2AYErDH7rNVpj61pw9tBvhslamaR+2LHn22ypTGHB7JsFWmvqWbySKfrTAlIUfvbIp8tsL0AObg6TxgK0xpzNFmQMBWmNKYg/dyRGrPFPO9HAcGAByrB2xzpTTm6E0EAV9JShY3omnOV5JaW2kX8JWk1pZ4BXwlqa2lLwFfSUpBDp/N+SpSW6uNQr6C1NZDT6KQryC19dCTKNQFKRfMqcoXtKcb8pWgFOTw2ZyvAiXru9A2eshXgtKYo2dzvhKUxBxto4d8JSiJOdpGD8+jJdKBAQAr0pCvIqULpsE2esRXklLFu2ivK+KrSCnI0aF6xFeQ0tsCwMtpxFeR0pijpxa+kpSqjUYjzleQUoiD5WjEV45SiKO7GkV81SgFuer+D4OcbUI0IHdcoINEtvKThHwMVp8xW/VJQg630GO28vMA5uCYJWarP2nM0c5WTGwdfcjm8/LtJuuyx6wVv8xZbB09MAzg2DFmK0mVwWJdIBOz1aQk5GOwDRCzFaU05OAUacxWlNKQo5dUvqKUhBw9l/MVpRTkcIWU8FWlNObgbFHCV5XSmINVacJXlZKYo4uMEkKVysv8Wtfd56oTzVMmvw3jIQDH6sn5KVK4GZOcnySFd9ZJ+GpSn8Ac7bwkfDUpBfkYHcTw1aQk5GBLN+GrSUnIwRNLyleSUpDDbYCUrySlMQfP5urIvLPCHBwnpmwLdQ9gDj4bIOWrQUnM0dZLqmtQjvu7DgwA2HhJz0+Qwo2X9PwUKdx4SflKUltbGqd8JSkFObrkZaxL0tVy+lblX+tStJ+aevZNzOYs6uvIUQDP72O+ItXaXt5qbwhDzMnWo2jI+WpUEnKwXBrzlajkUfZgY31MKNS+OEDkdTORUvW3onv+ymJJJXuRgmPJMV+ZSnZphC+pfGUqjTnYax/zlakk5mgPcsxXp9KYY4VqPCKEat6IrBPfivy7WNXdsVhVqaawKXa2iVWB8ZmPAtaajEd8pSvVYg18DHg84itdbW1ZGo/4alcaczTP+Zb4kpjDp3O+UtXS7o3xiK9SJdvawadzvkrV1vaN8YivUiUxBxdrxB7fKl+qz5oHprn6+2cEOXoF9fhqUJLl6BXUI7ad9sUaD1lZPvTYzeRfZuG+kMOAXlQ9XZee4zCAD3iMPTrPujcKNe9hABd1xJ6uXs/xbkAblB5bRRuSrWbRIRBbQUtCDi5XjT1dz67aQU5X2z/6ktXP1eeq5TDZ0KMAVrg+kYs9r1FAl9vEPlvVewBzcNDps5W9NOZoveWzzb0ewBy80Ppsc6/qKGnrwkmfbeqVhBy/hLJNvR7AHL2E8lWqJObwJZSvVCUxR5d1+LpW5YK5b+kSGrBNvZKQo70ANbOdEeTwqEU1TTgrzMFRS8BXhJKYo6OWgK8IJVtwoFdQviKUbP6A3akaB3w1qK3NH+KArwalMUevoHw1KIk5fAXlq0Gp9g/oFTTkq0EpyL0EPLUoE4gh5mR/DTTN+WpQCnJ0oBjylaB0ew3wAhrylaC2tjSJQ74alMQcHSiGfEUojbkPxpyvCCUxRyflQr4ilOpWAo8U+WpQCnLwcUpxxFeDkpCDg/OIrwSlIIcHihFfDUpijg5aIr4ilMQcbW9FfEUo1ZQHHbREfDUoBTm6kijiK0FJyNFBC18FSndYA6eeI74K1NpOghFfCUpijnYUY74alMQcHSjGfEUojTnYUYz5ilBbuzfGfDUoBTk6Uoz5SlBrG2bGfDUojTl6NucrQukmpWBLMearQm3t3hjrInTZwWUiStGJh0XLGXy00RXzVaMk5ODwJeErRsmOdOjwJeErRknM0aZLoovRVTNGUU3usrZ9qZvJl6L6zngI0G1JE6JJbyNa0S1X1of5ciRYNEcjhwFtyyRUk97zuRPQjfATtvo1ovvTgSFnK19pyNFLLlv1SkKOVq8J2xQqCTlcQLHVrAfmcnAkmbIVrTTmAThsTHXRun+oQN4VddUyHgG0fkp1Ccue9ei6gZTQrGfDej8ER5Ep2/yq2hhhXRSZEufHiNe5RLXvY3+TdRlf/NFnOaTEwTFngz861NGF63nhD7bH0oM52Ms8l3B0fOGHt3ZI+UpaGnNwoDnmK2lJzNE5wTHbPCyNOTr7NOYrYknM0a0dxmyrgtU+Z+sk1JhIsuaNyDrxufpRdP0BnBXfEUCLqDHfHCsJOXqeIXTrw3PW3mbVRDL/cjIrqq8cCjzIUYDvoxzzzbvSmKPDd76JVxJzePiuy9R1Pdm0aDvRXFaTyzwX8473dBNjp5tkRJx5ej6hDrqeLFGG0TnNPmBRlYz4CllLj+BIRnx1LHkERwqGnG/21dJTT5IRX+Fq66knyYhvdbCtp54kIyLL2rsF35piOhXNnWhmRcsifLf1IJRkxFe50pj7YMwJ5dqtaH+/eGzzpph3BW/qgwsqE49vztXSM4ASj69SJSEHx5OeLlTXNtmP+rt46OS/qvbh8e2BxWRDjQK4pinx+GpX8vArcB1T4vEVrzTm6DWVr3olMUeH8B5f9UofrIeez/kmWS09WC/xiLJgidjzQ56V5aMEhS/8+OWUb1mwrSe+JT5fiWrriW+J+qT380wpAV0w2HdAAg+uDU58XamyJzs6jvH56lJbj9pLfL661NKj9hKf2NL60He9upQB5MdZVpR88UfXEfh8JSp9CBw6juErUW09eC/x+WZT6YP30DwnsqnnMrd7AZj0AVEBLAVTwaDslz79EBxHqhvvnCYbtF5SpSMMMbf0xMkk4CtR6XPhwGtqwFei2noWXxLwTZ3aehZfEhBbVrl4vbYeDpcEfMUpjTk6duErTknM0V5vwDdpaukhiEnIN2dq64l8ieoSek6Yo2OXkK8UJTFHbxkI+WrRsa3TOV8pSkGOn875SlES8wAcnYdsU6Tq8HXrtsaEbEUoCTl6H0zIVoOSkMN3t4dsNegBzMEraMRWhMa+rTxXHVjPCnM0z9mK0JhsgwSOWiK2GpSEHE9zop/vg6jaRSOu6rpruyabMx6CAD27s9WkNObojFHEV5PSW77AkPPVpNbusouInaUPD0VVdA8PjKFHJzEiXZueDfQ+FvqYr0Sl9weAIeerUCnIU/DOIyWdB07wvH7Oups6X8zk12s/cWh/RI7CGE18vprV2r1IMdvEKY15isacr0ilSuzQW0tjvhqVLptG05yvSLW2kjTmmzmlyr3gUwvfxKmt1V4JX1FKV3uBPZiErSpV32xgBY3BmBOytO3qRtyIx8X0VrRtNhUcZKn6svowoJnPVpbSkMNneLay9ADm4DxeQrQ/emmKbjXbfCrkd2Y8AOCajYStSKUxT8GeY8JWpB7AHCxSEyqTuqqT+VwVXZGVEikWYQ3ZKRk921BJ1bMahRQ8CilbKZtS8w862ZeyVbIpVSMGh1wXslwgpzvFgCFnq1oPQA7OeKRsVSuNOXqzdco2mUpjjhZNKVuhegBzsGhK2QrV1NpGDikhVFt5qX//vl807gk475Hq0pQL1+n8HhbyMVsdqr6ZfZCz1aEk5GgdOtZ16LrOtxJN1olreZF80UfX947ZSlIach8MOVtFemBaB+dRx2wVqVqx7FtK2QpSEnL4Uqrr0b2lVO2Z4TsC8OWUba0vDTl6OWUrTA9M7djlNB3xVaa+nctpqmqNzwhy8HKajghlOm/qXLQtX+DBq2g64itKScixq2g6Ivoksec6fi3lK03prmBgyPlK09BWyNmmSseWNqRKR3y1KN26AR0y8hWjdOsG7F6w1OMrRumqOvDc4vFVozTm4LnF0+UoF8zp45HAkPMVohTkqiwcBjkhRP+zEM3bTdZlj1nLoB6A7JyBZjxfNUqXeaEndr5ylMQcHjRy1aP+yNLautTjqkdpyOGzOVc5egBycPzic1WjvspAWjexqE9iCLlvK+RctSgNOXou97lq0QOQo+dyrpW6/sjW1JzPVX8egBzcZCr1uepPf0SnQ8Ga3+erP0nM0Zrf5ytAbc1B+3wFKAk5ejoP+CpQS4+HSQO+CtTSXjqpOoSJIeS2ZkMDvgrU0hYAacBWgXq2pikCtgpUFe3YBzlbAapqduyDnK3+9CifBW3gBmzlp2etzRKw1Z805mibJWSrPz1bbZZQ15/Lorm8ns3rVlxWk5sme+o+zhj0jaZHAT3Bh2wlqWer8RKylaSercZLyFeS2mq8hMQpMK3YTOxZUfLFHz6x89Wn1hZFh3wFqrVF0SFbherb6jaGbAWqesU6yCNdn67F0vyNw2lqNPLodTRimyb1fVvJrmvSJdkn9UtV1tmkJ/zV213WcXAGyFGAE5+tTPVtLSSN2MpU39YeLxHbzKlvq/EbsVWmvq2WY8RWmPq2Wo4RX11qq+UY6bp0GTcu5ipq5Is9OlqMCYEqv1W76eH9pWh5xOyUGwYfBbZiNaDNMLDpG+tqlT3maNM3ZqtNA1u1acxWmwa2atOYrTYNbNWmMVttSkKO7lgfs9Wmga3aNGarTWnIwefupjHbpGlgqx+QsC3qDSgBCoecKOp9mJdZdS9Kkf/tD/KiwUevpImuRFfgZ3n38VXki+7vb4CR4KOtl0SXpOfDfPiBJAnRgfc80E8DMPp8hSqNOTieTNgq1dDWuseEOKpUvHaNXGK/ih+FeLmWYHHIcaiNQfaNgi5ed0fhS9aJVo7FU8F7FMAOfELkWs9sFALwMKRsJW5oayJE5b4YQm5rIkSd78EQclsrxlK2GVYS8hSsYVO2GVYacrBjlrIVrgcmFjTN+QpX+rhBsEGTss2xhrbmWFNCpj6IlRP/LWu//+2PqD4Ev6lpRj5s6rrbee2nJps/38qr6N/xfw== \ No newline at end of file diff --git a/scripts/import_diagram_containers.drawio b/scripts/import_diagram_containers.drawio new file mode 100644 index 00000000..7ee90533 --- /dev/null +++ b/scripts/import_diagram_containers.drawio @@ -0,0 +1 @@ +7ZpLc9owEMc/DTPtoR3LT7iGpumhPXHoMSOs9SMRFiPLPPrpK2EJLDuEZKZUDMQcsP67sqz9aWVLMAqmi80Dx8viFyNAR75HNqPg28j3/Unoyy+lbLUSoqRVcl6SVkMHYVb+AS16Wm1KArXlKBijolzaYsqqClJhaZhztrbdMkbtVpc4h4EwSzEdqr9LIgqtxp53MPyAMi9M076xLLDx1kJdYMLWHSm4HwVTzphozxabKVAVPhOYtt73I9b9nXGoxFsqyBAJXFbAH3GZFli09VeYNrqvWo6m1afEUz1gpKFQf9a3L7YmKJw1FQF1WTQK7tZFKWC2xKmyruU4kFohFlSbs5LSKaOM7+oGJIIxCaVeC86eoWOJd4eqIW90pltDptyODKTKw47rWKyAC9h0JB2IB2ALEHwrXbQ1NlD0uEx0cd1hHGqt6OCNtYb1sMr3Vz4EXp7o2J/k0MjGBhSUqBjE50KQZRCnqWsEyWRiIUBJ6IJBBlg0HGoFg8mcLVk1QGJ8vnZ8FCF0LkLgQ0awa0IIBXaWRLGTNNkjUtPTnA0nrj0f43DW9LkQOF5sw4knbuFU0AiOqewlPw6o66QgRdcNadJjNA7cMuIgX2xqgQUcR9TxUYTC6yaUeD1CjrNI8KYW8Aoe46DYBNfNBnljG07oO4FTA1+VqVyF9KHsDQrG5Fww5oiQzHMNI0I2C+QmUdaMP2dULev6MA6W3dPfP9vbc6Q+rnGEPRxfov+IA0gOj8jz/AGEODgRa8ZFwXJWYfqTsaUWn0CIrQ6QesW249/G2ay+o148VWJQPAd6h9PnfNeYtdRRxwtInprF0pQxT1+DUrOGp3BiIArMcxB9p+5CWwXtVbIcqFxarOyNh3+Ayey4dDCNz02pP+pdUeoCeBlRj6MzStEwmfxboXR8DfEyM/uB7A7Z+HaRvWn6uxBOMRpwQrfC6fQEeNiEdAYouWFA7537LgFX8IFrsJVy0TOg2Vr/IHaS2CXkV/xBy94Iu1hU4xt+ctlzm3NEsnj4LX1n6/wnIbj/Cw== \ No newline at end of file diff --git a/scripts/script_analyze_function_imports.py b/scripts/script_analyze_function_imports.py new file mode 100644 index 00000000..6a3118d2 --- /dev/null +++ b/scripts/script_analyze_function_imports.py @@ -0,0 +1,223 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Analyze function-level imports to determine which could be moved to header. + +Categories: +1. CIRCULAR - Import would cause circular dependency (must stay in function) +2. REDUNDANT - Same import already exists in header (can be removed) +3. MOVABLE - Could potentially be moved to header +""" + +import csv +from pathlib import Path +from typing import Dict, List, Set, Tuple +from collections import defaultdict + +# Paths +SCRIPT_DIR = Path(__file__).parent +INPUT_FILE = SCRIPT_DIR / "import_analysis.csv" +OUTPUT_FILE = SCRIPT_DIR / "function_imports_analysis.txt" + + +def _getContainer(moduleName: str) -> str: + """Extract container name from module path.""" + if moduleName == "gateway.app": + return "app" + + parts = moduleName.replace("gateway.", "").split(".") + if len(parts) < 2: + return "app" + + container = parts[1] + + # Skip tests and scripts + if container in ("tests", "scripts") or container.startswith("script_"): + return None + if parts[0] in ("tests", "scripts"): + return None + + # Handle features sub-containers + if container == "features" and len(parts) > 2: + return f"features.{parts[2]}" + + return container + + +def _analyzeImports() -> Tuple[Dict, Dict, Dict, Set]: + """ + Analyze imports and return: + - headerImports: Dict[module] -> Set[imported_modules] + - functionImports: Dict[module] -> List[(imported_module, function_name)] + - allModuleImports: Dict[module] -> Set[all_imports] (for circular detection) + - allModules: Set of all modules + """ + headerImports = defaultdict(set) + functionImports = defaultdict(list) + allModuleImports = defaultdict(set) + allModules = set() + + with open(INPUT_FILE, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + sourceFull = row["module_name"] + targetFull = row["imported_module_name"] + position = row["position"] + + # Skip external imports and relative imports + if not targetFull.startswith("modules."): + continue + if targetFull.startswith("(relative)"): + continue + + # Skip tests/scripts + sourceContainer = _getContainer(sourceFull) + if sourceContainer is None: + continue + + # Add gateway prefix for consistency + targetFull = f"gateway.{targetFull}" + + allModules.add(sourceFull) + allModules.add(targetFull) + allModuleImports[sourceFull].add(targetFull) + + if position == "header": + headerImports[sourceFull].add(targetFull) + else: + # Extract function name + funcName = position.replace("function ", "") + functionImports[sourceFull].append((targetFull, funcName)) + + return dict(headerImports), dict(functionImports), dict(allModuleImports), allModules + + +def _detectCircularDependency(source: str, target: str, allModuleImports: Dict) -> bool: + """ + Check if moving target import to header would create circular dependency. + Returns True if target already imports source (directly or indirectly). + """ + visited = set() + + def _canReach(current: str, goal: str) -> bool: + if current == goal: + return True + if current in visited: + return False + visited.add(current) + + for imported in allModuleImports.get(current, []): + if _canReach(imported, goal): + return True + return False + + # Check if target can reach source through its imports + return _canReach(target, source) + + +def main(): + """Main analysis function.""" + print("Analyzing function imports...") + headerImports, functionImports, allModuleImports, allModules = _analyzeImports() + + # Categorize function imports + circular = [] # Must stay in function (would cause circular import) + redundant = [] # Already imported in header (can be removed) + movable = [] # Could be moved to header + + totalFunctionImports = 0 + + for source, imports in sorted(functionImports.items()): + headerSet = headerImports.get(source, set()) + + for target, funcName in imports: + totalFunctionImports += 1 + + # Check if already in header + if target in headerSet: + redundant.append((source, target, funcName)) + continue + + # Check for circular dependency + if _detectCircularDependency(source, target, allModuleImports): + circular.append((source, target, funcName)) + continue + + # Otherwise, could be moved to header + movable.append((source, target, funcName)) + + # Generate report + lines = [] + lines.append("=" * 80) + lines.append("FUNCTION IMPORTS ANALYSIS") + lines.append("=" * 80) + lines.append(f"\nTotal function imports (internal modules): {totalFunctionImports}") + lines.append(f" - CIRCULAR (must stay): {len(circular):4}") + lines.append(f" - REDUNDANT (can remove): {len(redundant):4}") + lines.append(f" - MOVABLE (can move): {len(movable):4}") + + # Group movable by source module + movableBySource = defaultdict(list) + for source, target, funcName in movable: + movableBySource[source].append((target, funcName)) + + lines.append(f"\n\n{'=' * 80}") + lines.append("MOVABLE TO HEADER (grouped by source module)") + lines.append("These imports could potentially be moved to the module header.") + lines.append("=" * 80) + + for source in sorted(movableBySource.keys()): + imports = movableBySource[source] + lines.append(f"\n{source}") + lines.append("-" * len(source)) + for target, funcName in sorted(set(imports)): + shortTarget = target.replace("gateway.", "") + lines.append(f" [{funcName}] {shortTarget}") + + # Redundant imports + if redundant: + lines.append(f"\n\n{'=' * 80}") + lines.append("REDUNDANT IMPORTS (already in header - can be removed)") + lines.append("=" * 80) + + redundantBySource = defaultdict(list) + for source, target, funcName in redundant: + redundantBySource[source].append((target, funcName)) + + for source in sorted(redundantBySource.keys()): + imports = redundantBySource[source] + lines.append(f"\n{source}") + lines.append("-" * len(source)) + for target, funcName in sorted(set(imports)): + shortTarget = target.replace("gateway.", "") + lines.append(f" [{funcName}] {shortTarget}") + + # Circular imports (for reference) + if circular: + lines.append(f"\n\n{'=' * 80}") + lines.append("CIRCULAR DEPENDENCY (must stay in function)") + lines.append("=" * 80) + + circularBySource = defaultdict(list) + for source, target, funcName in circular: + circularBySource[source].append((target, funcName)) + + for source in sorted(circularBySource.keys()): + imports = circularBySource[source] + lines.append(f"\n{source}") + lines.append("-" * len(source)) + for target, funcName in sorted(set(imports)): + shortTarget = target.replace("gateway.", "") + lines.append(f" [{funcName}] {shortTarget}") + + # Write report + report = "\n".join(lines) + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + f.write(report) + + print(report) + print(f"\n\nReport saved to: {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/scripts/script_generate_container_diagram.py b/scripts/script_generate_container_diagram.py new file mode 100644 index 00000000..7f6243c9 --- /dev/null +++ b/scripts/script_generate_container_diagram.py @@ -0,0 +1,221 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Generate a simplified draw.io diagram showing container-to-container imports. +Aggregates all module imports into container-level relationships. +""" + +import csv +from pathlib import Path +from typing import Dict, Tuple +from collections import defaultdict +import html +import math + +# Paths +SCRIPT_DIR = Path(__file__).parent +INPUT_FILE = SCRIPT_DIR / "import_analysis.csv" +OUTPUT_FILE = SCRIPT_DIR / "import_diagram_containers.drawio" + +# Container colors +CONTAINER_COLORS = { + "app": "#dae8fc", # Light blue + "aichat": "#d5e8d4", # Light green + "auth": "#ffe6cc", # Light orange + "connectors": "#e1d5e7", # Light purple + "datamodels": "#fff2cc", # Light yellow + "interfaces": "#f8cecc", # Light red + "routes": "#d0cee2", # Light violet + "security": "#fad7ac", # Peach + "services": "#b1ddf0", # Sky blue + "shared": "#f0fff0", # Honeydew + "workflows": "#f5f5f5", # Light gray + "features": "#e2efda", # Sage green +} + + +def _getContainer(moduleName: str) -> str: + """Extract container name from module path.""" + if moduleName == "gateway.app": + return "app" + + parts = moduleName.replace("gateway.", "").split(".") + if len(parts) < 2: + return "app" + + container = parts[1] # modules.XXX or tests.XXX or scripts + + # Skip tests and scripts + if container in ("tests", "scripts") or container.startswith("script_"): + return None + if parts[0] in ("tests", "scripts"): + return None + + # Handle features sub-containers + if container == "features" and len(parts) > 2: + return f"features.{parts[2]}" + + return container + + +def _parseContainerImports() -> Tuple[Dict[str, int], Dict[Tuple[str, str], int]]: + """ + Parse import_analysis.csv and return: + - containerModuleCounts: Dict mapping container name to module count + - containerEdges: Dict mapping (source_container, target_container) to import count + """ + containerModules = defaultdict(set) + containerEdges = defaultdict(int) + + with open(INPUT_FILE, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + sourceFull = row["module_name"] + targetFull = row["imported_module_name"] + + # Skip external imports and relative imports + if not targetFull.startswith("modules."): + continue + if targetFull.startswith("(relative)"): + continue + + # Add gateway prefix to target for consistency + targetFull = f"gateway.{targetFull}" + + # Get containers + sourceContainer = _getContainer(sourceFull) + targetContainer = _getContainer(targetFull) + + # Skip if either is None (tests/scripts) + if sourceContainer is None or targetContainer is None: + continue + + # Track modules per container + containerModules[sourceContainer].add(sourceFull) + containerModules[targetContainer].add(targetFull) + + # Skip self-imports (within same container) + if sourceContainer == targetContainer: + continue + + # Count container-to-container imports + containerEdges[(sourceContainer, targetContainer)] += 1 + + # Convert module sets to counts + containerModuleCounts = {k: len(v) for k, v in containerModules.items()} + + return containerModuleCounts, dict(containerEdges) + + +def _generateDrawio(containerModuleCounts: Dict[str, int], + containerEdges: Dict[Tuple[str, str], int]) -> str: + """Generate draw.io XML content with container nodes and aggregated edges.""" + + containers = sorted(containerModuleCounts.keys()) + numContainers = len(containers) + + # Arrange containers in a circle for better visibility + centerX = 600 + centerY = 500 + radius = 400 + nodeWidth = 140 + nodeHeight = 60 + + # Calculate positions + containerPositions = {} + for i, container in enumerate(containers): + angle = (2 * math.pi * i / numContainers) - math.pi / 2 # Start from top + x = centerX + radius * math.cos(angle) - nodeWidth / 2 + y = centerY + radius * math.sin(angle) - nodeHeight / 2 + containerPositions[container] = (int(x), int(y)) + + cells = [] + + # Create container nodes + for container in containers: + x, y = containerPositions[container] + moduleCount = containerModuleCounts[container] + + # Get color + baseContainer = container.split(".")[0] + color = CONTAINER_COLORS.get(baseContainer, "#ffffff") + + # Create node + label = f"{container}\\n({moduleCount} modules)" + cells.append(f''' + + ''') + + # Create edges with import counts + edgeId = 1000 + for (source, target), count in sorted(containerEdges.items(), key=lambda x: -x[1]): + sourceId = f"container_{source.replace('.', '_')}" + targetId = f"container_{target.replace('.', '_')}" + + # Thicker line for more imports + strokeWidth = min(1 + count // 10, 5) + + cells.append(f''' + + ''') + edgeId += 1 + + # Assemble XML + xml = f''' + + + + + + +{chr(10).join(cells)} + + + +''' + + return xml + + +def main(): + """Main function.""" + print("Parsing container imports...") + containerModuleCounts, containerEdges = _parseContainerImports() + + print(f"Found {len(containerModuleCounts)} containers") + print(f"Found {len(containerEdges)} container-to-container relationships") + + # Print summary + print("\nContainer Import Summary:") + print("-" * 60) + + # Sort by total imports (outgoing) + outgoingCounts = defaultdict(int) + incomingCounts = defaultdict(int) + for (source, target), count in containerEdges.items(): + outgoingCounts[source] += count + incomingCounts[target] += count + + for container in sorted(containerModuleCounts.keys()): + modules = containerModuleCounts[container] + outgoing = outgoingCounts.get(container, 0) + incoming = incomingCounts.get(container, 0) + print(f" {container:25} {modules:3} modules | imports: {outgoing:4} out, {incoming:4} in") + + print("\nTop 15 Container Dependencies:") + print("-" * 60) + sortedEdges = sorted(containerEdges.items(), key=lambda x: -x[1])[:15] + for (source, target), count in sortedEdges: + print(f" {source:25} -> {target:25} : {count:4} imports") + + print("\nGenerating draw.io diagram...") + xml = _generateDrawio(containerModuleCounts, containerEdges) + + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + f.write(xml) + + print(f"\nDiagram saved to: {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/scripts/script_generate_import_diagram.py b/scripts/script_generate_import_diagram.py new file mode 100644 index 00000000..0a7dcdd9 --- /dev/null +++ b/scripts/script_generate_import_diagram.py @@ -0,0 +1,251 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Generate a draw.io diagram from import_analysis.csv +Shows all modules and their imports, grouped by container. +""" + +import csv +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Dict, List, Set, Tuple +from collections import defaultdict +import html + +# Paths +SCRIPT_DIR = Path(__file__).parent +INPUT_FILE = SCRIPT_DIR / "import_analysis.csv" +OUTPUT_FILE = SCRIPT_DIR / "import_diagram.drawio" + +# Container colors +CONTAINER_COLORS = { + "app": "#dae8fc", # Light blue + "aichat": "#d5e8d4", # Light green + "auth": "#ffe6cc", # Light orange + "connectors": "#e1d5e7", # Light purple + "datamodels": "#fff2cc", # Light yellow + "interfaces": "#f8cecc", # Light red + "routes": "#d0cee2", # Light violet + "security": "#fad7ac", # Peach + "services": "#b1ddf0", # Sky blue + "shared": "#d4edda", # Mint + "workflows": "#f5f5f5", # Light gray + "features": "#e2efda", # Sage green +} + +def _getContainer(moduleName: str) -> str: + """Extract container name from module path.""" + if moduleName == "gateway.app": + return "app" + + parts = moduleName.replace("gateway.", "").split(".") + if len(parts) < 2: + return "app" + + container = parts[1] # modules.XXX or tests.XXX or scripts + + # Skip tests and scripts + if container in ("tests", "scripts") or container.startswith("script_"): + return None + if parts[0] in ("tests", "scripts"): + return None + + # Handle features sub-containers + if container == "features" and len(parts) > 2: + return f"features.{parts[2]}" + + return container + +def _getShortName(moduleName: str) -> str: + """Get short display name for module.""" + parts = moduleName.replace("gateway.", "").split(".") + if moduleName == "gateway.app": + return "app" + # Return last part(s) for readability + if len(parts) > 2: + return ".".join(parts[-2:]) + return parts[-1] if parts else moduleName + +def _parseImports() -> Tuple[Dict[str, Set[str]], List[Tuple[str, str, str]]]: + """ + Parse import_analysis.csv and return: + - containers: Dict mapping container name to set of modules + - edges: List of (source, target, label) tuples + """ + containers = defaultdict(set) + edges = [] + seenEdges = set() + + with open(INPUT_FILE, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + sourceFull = row["module_name"] + targetFull = row["imported_module_name"] + position = row["position"] + + # Skip external imports and relative imports + if not targetFull.startswith("modules."): + continue + if targetFull.startswith("(relative)"): + continue + + # Add gateway prefix to target for consistency + targetFull = f"gateway.{targetFull}" + + # Get containers + sourceContainer = _getContainer(sourceFull) + targetContainer = _getContainer(targetFull) + + # Skip if either is None (tests/scripts) + if sourceContainer is None or targetContainer is None: + continue + + # Add to containers + containers[sourceContainer].add(sourceFull) + containers[targetContainer].add(targetFull) + + # Create edge key to avoid duplicates + edgeKey = (sourceFull, targetFull) + if edgeKey not in seenEdges: + seenEdges.add(edgeKey) + # Simplify label + label = "header" if position == "header" else position.replace("function ", "fn:") + edges.append((sourceFull, targetFull, label)) + + return dict(containers), edges + +def _generateDrawio(containers: Dict[str, Set[str]], edges: List[Tuple[str, str, str]]) -> str: + """Generate draw.io XML content.""" + + # Create module ID mapping + moduleIds = {} + idCounter = [2] # Start at 2 (0 and 1 are reserved) + + def _getModuleId(moduleName: str) -> str: + if moduleName not in moduleIds: + moduleIds[moduleName] = f"node_{idCounter[0]}" + idCounter[0] += 1 + return moduleIds[moduleName] + + # Calculate positions + containerX = 0 + containerY = 0 + containerWidth = 300 + containerHeight = 0 + containerPadding = 50 + moduleHeight = 30 + moduleWidth = 250 + modulePadding = 10 + + cells = [] + + # Sort containers + sortedContainers = sorted(containers.keys()) + + # Create container groups and modules + containerPositions = {} + currentY = 0 + currentX = 0 + maxHeightInRow = 0 + containersPerRow = 3 + containerCount = 0 + + for containerName in sortedContainers: + modules = sorted(containers[containerName]) + + # Calculate container size + numModules = len(modules) + height = numModules * (moduleHeight + modulePadding) + 60 + + # Position container + if containerCount > 0 and containerCount % containersPerRow == 0: + currentY += maxHeightInRow + containerPadding + currentX = 0 + maxHeightInRow = 0 + + containerPositions[containerName] = (currentX, currentY) + maxHeightInRow = max(maxHeightInRow, height) + + # Get color + baseContainer = containerName.split(".")[0] + color = CONTAINER_COLORS.get(baseContainer, "#ffffff") + + # Create container (swimlane) + containerId = f"container_{containerName.replace('.', '_')}" + cells.append(f''' + + ''') + + # Create module nodes inside container + moduleY = 30 + for moduleName in modules: + moduleId = _getModuleId(moduleName) + shortName = _getShortName(moduleName) + cells.append(f''' + + ''') + moduleY += moduleHeight + modulePadding + + currentX += containerWidth + containerPadding + containerCount += 1 + + # Create edges (only between different containers to reduce clutter) + edgeId = idCounter[0] + for source, target, label in edges: + sourceContainer = _getContainer(source) + targetContainer = _getContainer(target) + + # Skip internal container edges for clarity + if sourceContainer == targetContainer: + continue + + sourceId = _getModuleId(source) + targetId = _getModuleId(target) + + # Shorten label if too long + displayLabel = label[:20] + "..." if len(label) > 20 else label + + cells.append(f''' + + ''') + edgeId += 1 + + # Assemble XML + xml = f''' + + + + + + +{chr(10).join(cells)} + + + +''' + + return xml + +def main(): + """Main function.""" + print("Parsing imports...") + containers, edges = _parseImports() + + print(f"Found {len(containers)} containers with {sum(len(m) for m in containers.values())} modules") + print(f"Found {len(edges)} unique import edges") + + print("Generating draw.io diagram...") + xml = _generateDrawio(containers, edges) + + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + f.write(xml) + + print(f"Diagram saved to: {OUTPUT_FILE}") + + # Print container summary + print("\nContainers:") + for name, modules in sorted(containers.items()): + print(f" {name}: {len(modules)} modules") + +if __name__ == "__main__": + main() diff --git a/scripts/script_remove_redundant_imports.py b/scripts/script_remove_redundant_imports.py new file mode 100644 index 00000000..1c83eb9e --- /dev/null +++ b/scripts/script_remove_redundant_imports.py @@ -0,0 +1,245 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Remove redundant function-level imports that already exist in the header. +""" + +import csv +import re +import ast +from pathlib import Path +from typing import Dict, List, Set, Tuple +from collections import defaultdict + +# Paths +SCRIPT_DIR = Path(__file__).parent +GATEWAY_ROOT = SCRIPT_DIR.parent +INPUT_FILE = SCRIPT_DIR / "import_analysis.csv" + + +def _getContainer(moduleName: str) -> str: + """Extract container name from module path.""" + if moduleName == "gateway.app": + return "app" + + parts = moduleName.replace("gateway.", "").split(".") + if len(parts) < 2: + return "app" + + container = parts[1] + + if container in ("tests", "scripts") or container.startswith("script_"): + return None + if parts[0] in ("tests", "scripts"): + return None + + if container == "features" and len(parts) > 2: + return f"features.{parts[2]}" + + return container + + +def _findRedundantImports() -> Dict[str, List[Tuple[str, str]]]: + """ + Find redundant function imports (already in header). + Returns: Dict[source_module] -> List[(target_module, function_name)] + """ + headerImports = defaultdict(set) + functionImports = defaultdict(list) + + with open(INPUT_FILE, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + sourceFull = row["module_name"] + targetFull = row["imported_module_name"] + position = row["position"] + + if not targetFull.startswith("modules."): + continue + if targetFull.startswith("(relative)"): + continue + + sourceContainer = _getContainer(sourceFull) + if sourceContainer is None: + continue + + targetFull = f"gateway.{targetFull}" + + if position == "header": + headerImports[sourceFull].add(targetFull) + else: + funcName = position.replace("function ", "") + functionImports[sourceFull].append((targetFull, funcName)) + + # Find redundant + redundant = defaultdict(list) + for source, imports in functionImports.items(): + headerSet = headerImports.get(source, set()) + for target, funcName in imports: + if target in headerSet: + redundant[source].append((target, funcName)) + + return dict(redundant) + + +def _modulePathToFilePath(moduleName: str) -> Path: + """Convert module name to file path.""" + # gateway.modules.xxx.yyy -> modules/xxx/yyy.py + parts = moduleName.replace("gateway.", "").split(".") + filePath = GATEWAY_ROOT + for part in parts: + filePath = filePath / part + return filePath.with_suffix(".py") + + +def _removeImportFromFunction(filePath: Path, targetModule: str, funcName: str) -> bool: + """ + Remove a specific import statement from inside a function. + Returns True if successful. + """ + if not filePath.exists(): + print(f" File not found: {filePath}") + return False + + with open(filePath, "r", encoding="utf-8") as f: + content = f.read() + + # The import we're looking for (without gateway prefix) + importModule = targetModule.replace("gateway.", "") + + # Build regex patterns for different import styles + patterns = [ + # from modules.xxx import yyy + rf'(\n[ \t]+)(from {re.escape(importModule)} import [^\n]+)', + # import modules.xxx + rf'(\n[ \t]+)(import {re.escape(importModule)}[^\n]*)', + # from modules.xxx.yyy import zzz (partial match) + rf'(\n[ \t]+)(from {re.escape(importModule.rsplit(".", 1)[0])} import [^\n]*{re.escape(importModule.rsplit(".", 1)[-1])}[^\n]*)', + ] + + modified = False + for pattern in patterns: + matches = list(re.finditer(pattern, content)) + for match in matches: + # Check if this import is inside the target function + # by looking backwards for the function definition + startPos = match.start() + beforeMatch = content[:startPos] + + # Find the most recent function definition + funcPattern = rf'def {re.escape(funcName)}\s*\(' + funcMatches = list(re.finditer(funcPattern, beforeMatch)) + + if funcMatches: + lastFuncStart = funcMatches[-1].start() + # Check there's no other function definition between the func and the import + betweenText = beforeMatch[lastFuncStart:] + otherFuncs = re.findall(r'\ndef [a-zA-Z_][a-zA-Z0-9_]*\s*\(', betweenText) + + if len(otherFuncs) <= 1: # Only our target function + # Remove this import line + indent = match.group(1) + importLine = match.group(2) + fullMatch = match.group(0) + content = content[:match.start()] + content[match.end():] + modified = True + print(f" Removed: {importLine.strip()}") + break + + if modified: + break + + if modified: + with open(filePath, "w", encoding="utf-8") as f: + f.write(content) + return True + + return False + + +def _removeImportsWithAst(filePath: Path, redundantImports: List[Tuple[str, str]]) -> int: + """ + Use AST to properly identify and remove redundant imports. + Returns count of removed imports. + """ + if not filePath.exists(): + return 0 + + with open(filePath, "r", encoding="utf-8") as f: + lines = f.readlines() + + content = "".join(lines) + + try: + tree = ast.parse(content) + except SyntaxError: + print(f" Syntax error in {filePath}") + return 0 + + # Group by function + importsByFunc = defaultdict(set) + for target, funcName in redundantImports: + importModule = target.replace("gateway.", "") + importsByFunc[funcName].add(importModule) + + # Find imports inside functions + linesToRemove = set() + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + funcName = node.name + if funcName not in importsByFunc: + continue + + targetModules = importsByFunc[funcName] + + # Walk the function body for imports + for child in ast.walk(node): + if isinstance(child, ast.ImportFrom): + if child.module and child.module in targetModules: + linesToRemove.add(child.lineno) + elif isinstance(child, ast.Import): + for alias in child.names: + if alias.name in targetModules: + linesToRemove.add(child.lineno) + + if not linesToRemove: + return 0 + + # Remove the lines + newLines = [] + for i, line in enumerate(lines, 1): + if i not in linesToRemove: + newLines.append(line) + else: + print(f" Line {i}: {line.strip()}") + + with open(filePath, "w", encoding="utf-8") as f: + f.writelines(newLines) + + return len(linesToRemove) + + +def main(): + """Main function.""" + print("Finding redundant imports...") + redundant = _findRedundantImports() + + totalCount = sum(len(v) for v in redundant.values()) + print(f"Found {totalCount} redundant imports in {len(redundant)} files\n") + + removedCount = 0 + + for source, imports in sorted(redundant.items()): + filePath = _modulePathToFilePath(source) + print(f"\n{source}") + print(f" File: {filePath}") + + removed = _removeImportsWithAst(filePath, imports) + removedCount += removed + + print(f"\n\nTotal removed: {removedCount} imports") + + +if __name__ == "__main__": + main() diff --git a/tests/functional/test01_ai_model_selection.py b/tests/functional/test01_ai_model_selection.py index 1d4f2d49..b06e9c64 100644 --- a/tests/functional/test01_ai_model_selection.py +++ b/tests/functional/test01_ai_model_selection.py @@ -29,8 +29,8 @@ from modules.datamodels.datamodelAi import ( ProcessingModeEnum, ) from modules.datamodels.datamodelUam import User -from modules.aichat.aicore.aicoreModelRegistry import modelRegistry -from modules.aichat.aicore.aicoreModelSelector import modelSelector +from modules.aicore.aicoreModelRegistry import modelRegistry +from modules.aicore.aicoreModelSelector import modelSelector class ModelSelectionTester: @@ -46,7 +46,7 @@ class ModelSelectionTester: self.services = getServices(testUser, None) async def initialize(self) -> None: - from modules.aichat.serviceAi.mainServiceAi import AiService + from modules.services.serviceAi.mainServiceAi import AiService from modules.interfaces.interfaceAiObjects import AiObjects self.services.ai = await AiService.create(self.services) diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 2f348595..12a374f8 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -68,24 +68,24 @@ class AIModelsTester: logging.getLogger().setLevel(logging.DEBUG) # Initialize the model registry with all connectors - from modules.aichat.aicore.aicoreModelRegistry import modelRegistry - from modules.aichat.aicore.aicorePluginTavily import AiTavily - from modules.aichat.aicore.aicorePluginPerplexity import AiPerplexity + from modules.aicore.aicoreModelRegistry import modelRegistry + from modules.aicore.aicorePluginTavily import AiTavily + from modules.aicore.aicorePluginPerplexity import AiPerplexity # Note: We don't need to register web connectors for IMAGE_ANALYSE testing # modelRegistry.registerConnector(AiTavily()) # modelRegistry.registerConnector(AiPerplexity()) # The AI service needs to be recreated with proper initialization - from modules.aichat.serviceAi.mainServiceAi import AiService + from modules.services.serviceAi.mainServiceAi import AiService self.services.ai = await AiService.create(self.services) # Also initialize extraction service for image processing - from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService + from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( @@ -311,7 +311,7 @@ class AIModelsTester: print(f"{'='*60}") # Get model from registry - from modules.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.aicore.aicoreModelRegistry import modelRegistry model = modelRegistry.getModel(modelName) if not model: @@ -693,7 +693,7 @@ Width: {crawlWidth} def getAllAvailableModels(self) -> List[Dict[str, Any]]: """Get all available models with their supported operation types.""" - from modules.aichat.aicore.aicoreModelRegistry import modelRegistry + from modules.aicore.aicoreModelRegistry import modelRegistry from modules.datamodels.datamodelAi import OperationTypeEnum # Get all models from registry diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index de891238..a80be79c 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -94,7 +94,7 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) # Import and initialize services @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.aichat.datamodelFeatureAiChat import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.aichat.datamodelFeatureAiChat import ChatMessage + from modules.datamodels.datamodelChat import ChatMessage import time testMessage = ChatMessage( @@ -201,7 +201,7 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,7 +283,7 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat interfaceDbChat = interfaceDbChat.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 4cf21e10..657946a9 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,10 +42,10 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat + import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat currentTimestamp = time.time() diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 501204ad..fac1ab41 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,10 +20,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat class WorkflowWithDocumentsTester: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index e2138a6b..4b39454a 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,10 +22,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat class WorkflowPromptVariationsTester: diff --git a/tests/functional/test07_json_merge.py b/tests/functional/test07_json_merge.py index 897c6a4f..de70052f 100644 --- a/tests/functional/test07_json_merge.py +++ b/tests/functional/test07_json_merge.py @@ -11,7 +11,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument # type: ignore diff --git a/tests/functional/test08_json_finalization.py b/tests/functional/test08_json_finalization.py index f6345150..a05daccc 100644 --- a/tests/functional/test08_json_finalization.py +++ b/tests/functional/test08_json_finalization.py @@ -32,7 +32,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import after path setup -from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore +from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore from modules.shared.jsonUtils import extractSectionsFromDocument, extractJsonString, repairBrokenJson # type: ignore diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 5ecf5a1f..a6f99236 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index fb1ec6c7..e1990910 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat class DocumentGenerationFormatsTester10: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 0c58f8ce..43c294e4 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,10 +23,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.aichat.datamodelFeatureAiChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.workflows.automation import chatStart -import modules.aichat.interfaceFeatureAiChat as interfaceFeatureAiChat +import modules.interfaces.interfaceDbChat as interfaceFeatureAiChat class CodeGenerationFormatsTester11: diff --git a/tests/functional/test12_json_split_merge.py b/tests/functional/test12_json_split_merge.py index ec882824..4dac56cb 100644 --- a/tests/functional/test12_json_split_merge.py +++ b/tests/functional/test12_json_split_merge.py @@ -20,7 +20,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) # Import JSON merger from workflow tools -from modules.aichat.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger +from modules.services.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger from modules.shared.jsonContinuation import getContexts diff --git a/tests/functional/test_kpi_full.py b/tests/functional/test_kpi_full.py index c25f48bc..32e40f07 100644 --- a/tests/functional/test_kpi_full.py +++ b/tests/functional/test_kpi_full.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState # Load actual JSON response diff --git a/tests/functional/test_kpi_incomplete.py b/tests/functional/test_kpi_incomplete.py index d7170f48..f6e60aa0 100644 --- a/tests/functional/test_kpi_incomplete.py +++ b/tests/functional/test_kpi_incomplete.py @@ -11,7 +11,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler from modules.datamodels.datamodelAi import JsonAccumulationState from modules.shared.jsonUtils import extractJsonString, repairBrokenJson diff --git a/tests/functional/test_kpi_path.py b/tests/functional/test_kpi_path.py index 824b145c..0c54e3c2 100644 --- a/tests/functional/test_kpi_path.py +++ b/tests/functional/test_kpi_path.py @@ -10,7 +10,7 @@ _gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ". if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) -from modules.aichat.serviceAi.subJsonResponseHandling import JsonResponseHandler +from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # Test JSON matching the actual response test_json = { diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index 50fafe83..a2b69576 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/services/test_json_extraction_merging.py b/tests/unit/services/test_json_extraction_merging.py index 100e3f67..07ecfa4b 100644 --- a/tests/unit/services/test_json_extraction_merging.py +++ b/tests/unit/services/test_json_extraction_merging.py @@ -14,7 +14,7 @@ import os sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../..')) from modules.datamodels.datamodelExtraction import ContentPart -from modules.aichat.serviceExtraction.mainServiceExtraction import ExtractionService +from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService def test_detects_json_with_code_fences(): diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index afa873a8..ae502397 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 1bde895f..09f6e92c 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.aichat.datamodelFeatureAiChat import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel From f45cf794744a42a4df9c9773bf21eaae0deb6f80 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 23 Jan 2026 07:35:53 +0100 Subject: [PATCH 16/32] fixes --- modules/features/aichat/mainAiChat.py | 166 ++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 modules/features/aichat/mainAiChat.py diff --git a/modules/features/aichat/mainAiChat.py b/modules/features/aichat/mainAiChat.py new file mode 100644 index 00000000..2e6514e6 --- /dev/null +++ b/modules/features/aichat/mainAiChat.py @@ -0,0 +1,166 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +AIChat Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. + +AIChat is the dynamic chat workflow feature that handles: +- AI-powered document processing +- Dynamic workflow execution +- Automation definitions +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "chatworkflow" +FEATURE_LABEL = {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"} +FEATURE_ICON = "mdi-message-cog" + +# UI Objects for RBAC catalog +UI_OBJECTS = [ + { + "objectKey": "ui.feature.aichat.workflows", + "label": {"en": "Workflows", "de": "Workflows", "fr": "Workflows"}, + "meta": {"area": "workflows"} + }, + { + "objectKey": "ui.feature.aichat.automations", + "label": {"en": "Automations", "de": "Automatisierungen", "fr": "Automatisations"}, + "meta": {"area": "automations"} + }, + { + "objectKey": "ui.feature.aichat.logs", + "label": {"en": "Logs", "de": "Logs", "fr": "Journaux"}, + "meta": {"area": "logs"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.aichat.workflow.start", + "label": {"en": "Start Workflow", "de": "Workflow starten", "fr": "Démarrer workflow"}, + "meta": {"endpoint": "/api/chat/playground/start", "method": "POST"} + }, + { + "objectKey": "resource.feature.aichat.workflow.stop", + "label": {"en": "Stop Workflow", "de": "Workflow stoppen", "fr": "Arrêter workflow"}, + "meta": {"endpoint": "/api/chat/playground/stop/{workflowId}", "method": "POST"} + }, + { + "objectKey": "resource.feature.aichat.workflow.delete", + "label": {"en": "Delete Workflow", "de": "Workflow löschen", "fr": "Supprimer workflow"}, + "meta": {"endpoint": "/api/chat/playground/workflow/{workflowId}", "method": "DELETE"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "workflow-admin", + "description": { + "en": "Workflow Administrator - Full access to workflow configuration and execution", + "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", + "fr": "Administrateur workflow - Accès complet à la configuration et exécution" + } + }, + { + "roleLabel": "workflow-editor", + "description": { + "en": "Workflow Editor - Create and modify workflows", + "de": "Workflow-Editor - Workflows erstellen und bearbeiten", + "fr": "Éditeur workflow - Créer et modifier les workflows" + } + }, + { + "roleLabel": "workflow-viewer", + "description": { + "en": "Workflow Viewer - View workflows and execution results", + "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", + "fr": "Visualiseur workflow - Consulter les workflows et résultats" + } + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """ + Register this feature's RBAC objects in the catalog. + + Args: + catalogService: The RBAC catalog service instance + + Returns: + True if registration was successful + """ + try: + # Register UI objects + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + # Register Resource objects + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False + + +async def onStart(eventUser) -> None: + """ + Called when the feature container starts. + Initializes AI connectors for model registry. + """ + try: + from modules.aicore.aicoreModelRegistry import modelRegistry + modelRegistry.ensureConnectorsRegistered() + logger.info(f"Feature '{FEATURE_CODE}' started - AI connectors initialized") + except Exception as e: + logger.error(f"Feature '{FEATURE_CODE}' failed to initialize AI connectors: {e}") + + +async def onStop(eventUser) -> None: + """Called when the feature container stops.""" + logger.info(f"Feature '{FEATURE_CODE}' stopped") From 6187cc7295041cd30e61817e0f195075146871f8 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 23 Jan 2026 21:05:47 +0100 Subject: [PATCH 17/32] refactored pages ui access with saas mandates --- modules/datamodels/__init__.py | 1 - .../features/trustee/routeFeatureTrustee.py | 35 ++- modules/routes/routeAdminFeatures.py | 2 +- modules/routes/routeChat.py | 2 +- modules/routes/routeSecurityLocal.py | 2 +- modules/shared/jsonUtils.py | 5 +- scripts/function_imports_analysis.txt | 249 +++++++++--------- scripts/import_analysis.csv | 3 + 8 files changed, 168 insertions(+), 131 deletions(-) diff --git a/modules/datamodels/__init__.py b/modules/datamodels/__init__.py index 7c45ab08..4bc577ee 100644 --- a/modules/datamodels/__init__.py +++ b/modules/datamodels/__init__.py @@ -10,7 +10,6 @@ Usage examples: from . import datamodelAi as ai from . import datamodelUam as uam from . import datamodelSecurity as security -from . import datamodelNeutralizer as neutralizer from . import datamodelChat as chat from . import datamodelFiles as files from . import datamodelVoice as voice diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 196c8b18..e05f6dc0 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -154,13 +154,28 @@ async def getRoleOptions( async def getContractOptions( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), + organisationId: Optional[str] = Query(None, description="Optional: Filter by organisation ID"), context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: - """Get contract options for select dropdowns. Returns: [{ value, label }]""" + """ + Get contract options for select dropdowns. + + Optionally filter by organisationId to get only contracts for a specific organisation. + This is useful for dependent dropdowns in forms. + + Returns: [{ value, label }] + """ mandateId = await _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - result = interface.getAllContracts(None) - items = result.items if hasattr(result, 'items') else result + + if organisationId: + # Gefiltert nach Organisation + items = interface.getContractsByOrganisation(organisationId) + else: + # Alle Contracts + result = interface.getAllContracts(None) + items = result.items if hasattr(result, 'items') else result + return [{"value": c.id, "label": c.label or c.name or c.id} for c in items] @@ -176,7 +191,7 @@ async def getDocumentOptions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllDocuments(None) items = result.items if hasattr(result, 'items') else result - return [{"value": d.id, "label": d.name or d.id} for d in items] + return [{"value": d.id, "label": d.documentName or d.id} for d in items] @router.get("/{instanceId}/positions/options", response_model=List[Dict[str, Any]]) @@ -191,7 +206,17 @@ async def getPositionOptions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(None) items = result.items if hasattr(result, 'items') else result - return [{"value": p.id, "label": p.title or p.id} for p in items] + # Erstelle Label aus Datum, Firma und Beschreibung + def _makePositionLabel(p): + parts = [] + if p.valuta: + parts.append(str(p.valuta)[:10]) # Datum ohne Zeit + if p.company: + parts.append(p.company[:30]) + if p.desc: + parts.append(p.desc[:30]) + return " - ".join(parts) if parts else p.id + return [{"value": p.id, "label": _makePositionLabel(p)} for p in items] # ============================================================================ diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index d5f1a8f5..575af01e 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -17,7 +17,7 @@ import logging from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdmin -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelFeatures import Feature, FeatureInstance from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface diff --git a/modules/routes/routeChat.py b/modules/routes/routeChat.py index 88d4a4b1..22aa764e 100644 --- a/modules/routes/routeChat.py +++ b/modules/routes/routeChat.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -from . import interfaceFeatureAiChat as interfaceDbChat +from modules.interfaces import interfaceDbChat # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 41fe2cab..4c61cbc1 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -17,7 +17,7 @@ from jose import jwt from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie from modules.interfaces.interfaceDbApp import getInterface, getRootInterface -from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority +from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG diff --git a/modules/shared/jsonUtils.py b/modules/shared/jsonUtils.py index 1fa5d30d..37c1ae36 100644 --- a/modules/shared/jsonUtils.py +++ b/modules/shared/jsonUtils.py @@ -5,7 +5,6 @@ import logging import re from typing import Any, Dict, List, Optional, Tuple, Union, Type, TypeVar from pydantic import BaseModel, ValidationError -from modules.datamodels.datamodelAi import ContinuationContext logger = logging.getLogger(__name__) @@ -867,7 +866,7 @@ def buildContinuationContext( lastRawResponse: Optional[str] = None, useCaseId: Optional[str] = None, templateStructure: Optional[str] = None -) -> ContinuationContext: +) -> "ContinuationContext": """ Build context information from accumulated sections for continuation prompt. @@ -882,6 +881,8 @@ def buildContinuationContext( Returns: ContinuationContext: Pydantic model with all continuation context information """ + # Lazy import to avoid circular dependency + from modules.datamodels.datamodelAi import ContinuationContext section_count = len(allSections) # Build summary of delivered data (per-section counts) diff --git a/scripts/function_imports_analysis.txt b/scripts/function_imports_analysis.txt index 23f26c9f..a3909ed4 100644 --- a/scripts/function_imports_analysis.txt +++ b/scripts/function_imports_analysis.txt @@ -2,10 +2,10 @@ FUNCTION IMPORTS ANALYSIS ================================================================================ -Total function imports (internal modules): 226 +Total function imports (internal modules): 229 - CIRCULAR (must stay): 4 - REDUNDANT (can remove): 0 - - MOVABLE (can move): 222 + - MOVABLE (can move): 225 ================================================================================ @@ -17,121 +17,6 @@ gateway.app ----------- [lifespan] modules.shared.auditLogger -gateway.modules.aichat.datamodelFeatureAiChat ---------------------------------------------- - [updateFromSelection] modules.datamodels.datamodelWorkflow - -gateway.modules.aichat.interfaceFeatureAiChat ---------------------------------------------- - [_enrichAutomationsWithUserAndMandate] modules.interfaces.interfaceDbApp - [storeDebugMessageAndDocuments] modules.interfaces.interfaceDbManagement - [setUserContext] modules.security.rootAccess - [_notifyAutomationChanged] modules.shared.callbackRegistry - [storeDebugMessageAndDocuments] modules.shared.debugLogger - -gateway.modules.aichat.serviceAi.mainServiceAi ----------------------------------------------- - [renderResult] modules.aichat.serviceGeneration.mainServiceGeneration - [_handleCodeGeneration] modules.aichat.serviceGeneration.paths.codePath - [_handleDocumentGeneration] modules.aichat.serviceGeneration.paths.documentPath - [_handleImageGeneration] modules.aichat.serviceGeneration.paths.imagePath - -gateway.modules.aichat.serviceAi.subContentExtraction ------------------------------------------------------ - [extractTextFromImage] modules.datamodels.datamodelAi - [processTextContentWithAi] modules.datamodels.datamodelAi - -gateway.modules.aichat.serviceAi.subJsonResponseHandling --------------------------------------------------------- - [mergeFragmentIntoSection] modules.shared.debugLogger - -gateway.modules.aichat.serviceAi.subStructureFilling ----------------------------------------------------- - [_getAcceptedSectionTypesForFormat] modules.aichat.serviceGeneration.renderers.registry - [_getAcceptedSectionTypesForFormat] modules.datamodels.datamodelJson - [buildSectionPromptWithContinuation] modules.shared.jsonContinuation - [_extractAndMergeMultipleJsonBlocks] modules.shared.jsonUtils - [_processAiResponseForSection] modules.shared.jsonUtils - [_processSingleSection] modules.shared.jsonUtils - -gateway.modules.aichat.serviceAi.subStructureGeneration -------------------------------------------------------- - [generateStructure] modules.aichat.serviceGeneration.renderers.registry - [generateStructure] modules.shared - [generateStructure] modules.shared.jsonContinuation - -gateway.modules.aichat.serviceExtraction.mainServiceExtraction --------------------------------------------------------------- - [extractContent] modules.interfaces.interfaceDbManagement - [extractContent] modules.shared.debugLogger - -gateway.modules.aichat.serviceExtraction.subPromptBuilderExtraction -------------------------------------------------------------------- - [buildExtractionPrompt] modules.shared.debugLogger - -gateway.modules.aichat.serviceGeneration.mainServiceGeneration --------------------------------------------------------------- - [getAdaptiveExtractionPrompt] modules.aichat.serviceExtraction.subPromptBuilderExtraction - [renderReport] modules.aichat.serviceGeneration.renderers.registry - [generateDocumentWithTwoPhases] modules.aichat.serviceGeneration.subContentGenerator - [generateDocumentWithTwoPhases] modules.aichat.serviceGeneration.subStructureGenerator - -gateway.modules.aichat.serviceGeneration.paths.codePath -------------------------------------------------------- - [_getCodeRenderer] modules.aichat.serviceGeneration.renderers.registry - [generateCode] modules.datamodels.datamodelDocument - [_generateCodeStructure] modules.shared.jsonContinuation - [_generateSingleFileContent] modules.shared.jsonContinuation - -gateway.modules.aichat.serviceGeneration.renderers.rendererDocx ---------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererHtml ---------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererImage ----------------------------------------------------------------- - [_compressPromptWithAi] modules.datamodels.datamodelAi - [_generateAiImage] modules.datamodels.datamodelAi - -gateway.modules.aichat.serviceGeneration.renderers.rendererJson ---------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererMarkdown -------------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererPdf --------------------------------------------------------------- - [_getAiStylesWithPdfColors] modules.datamodels.datamodelAi - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererPptx ---------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererText ---------------------------------------------------------------- - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.renderers.rendererXlsx ---------------------------------------------------------------- - [_getAiStylesWithExcelColors] modules.datamodels.datamodelAi - [getAcceptedSectionTypes] modules.datamodels.datamodelJson - -gateway.modules.aichat.serviceGeneration.subContentGenerator ------------------------------------------------------------- - [_generateImageSection] modules.datamodels.datamodelAi - [_generateSimpleSection] modules.datamodels.datamodelAi - [_generateSimpleSection] modules.shared.jsonUtils - -gateway.modules.aichat.serviceGeneration.subStructureGenerator --------------------------------------------------------------- - [generateStructure] modules.datamodels.datamodelAi - gateway.modules.auth.authentication ----------------------------------- [requireSysAdmin] modules.shared.auditLogger @@ -150,6 +35,14 @@ gateway.modules.auth.tokenRefreshService [proactive_refresh] modules.security.rootAccess [refresh_expired_tokens] modules.security.rootAccess +gateway.modules.datamodels.datamodelChat +---------------------------------------- + [updateFromSelection] modules.datamodels.datamodelWorkflow + +gateway.modules.features.aichat.mainAiChat +------------------------------------------ + [onStart] modules.aicore.aicoreModelRegistry + gateway.modules.features.automation.routeFeatureAutomation ---------------------------------------------------------- [execute_automation] modules.services @@ -197,6 +90,14 @@ gateway.modules.interfaces.interfaceDbApp ----------------------------------------- [getRootInterface] modules.security.rootAccess +gateway.modules.interfaces.interfaceDbChat +------------------------------------------ + [_enrichAutomationsWithUserAndMandate] modules.interfaces.interfaceDbApp + [storeDebugMessageAndDocuments] modules.interfaces.interfaceDbManagement + [setUserContext] modules.security.rootAccess + [_notifyAutomationChanged] modules.shared.callbackRegistry + [storeDebugMessageAndDocuments] modules.shared.debugLogger + gateway.modules.interfaces.interfaceDbManagement ------------------------------------------------ [_initializeStandardPrompts] modules.interfaces.interfaceDbApp @@ -313,15 +214,123 @@ gateway.modules.security.rootAccess gateway.modules.services.__init__ --------------------------------- [__init__] modules.interfaces.interfaceDbApp + [__init__] modules.interfaces.interfaceDbChat [__init__] modules.interfaces.interfaceDbManagement +gateway.modules.services.serviceAi.mainAiChat +--------------------------------------------- + [onStart] modules.aicore.aicoreModelRegistry + +gateway.modules.services.serviceAi.mainServiceAi +------------------------------------------------ + [renderResult] modules.services.serviceGeneration.mainServiceGeneration + [_handleCodeGeneration] modules.services.serviceGeneration.paths.codePath + [_handleDocumentGeneration] modules.services.serviceGeneration.paths.documentPath + [_handleImageGeneration] modules.services.serviceGeneration.paths.imagePath + +gateway.modules.services.serviceAi.subContentExtraction +------------------------------------------------------- + [extractTextFromImage] modules.datamodels.datamodelAi + [processTextContentWithAi] modules.datamodels.datamodelAi + +gateway.modules.services.serviceAi.subJsonResponseHandling +---------------------------------------------------------- + [mergeFragmentIntoSection] modules.shared.debugLogger + +gateway.modules.services.serviceAi.subStructureFilling +------------------------------------------------------ + [_getAcceptedSectionTypesForFormat] modules.datamodels.datamodelJson + [_getAcceptedSectionTypesForFormat] modules.services.serviceGeneration.renderers.registry + [buildSectionPromptWithContinuation] modules.shared.jsonContinuation + [_extractAndMergeMultipleJsonBlocks] modules.shared.jsonUtils + [_processAiResponseForSection] modules.shared.jsonUtils + [_processSingleSection] modules.shared.jsonUtils + +gateway.modules.services.serviceAi.subStructureGeneration +--------------------------------------------------------- + [generateStructure] modules.services.serviceGeneration.renderers.registry + [generateStructure] modules.shared + [generateStructure] modules.shared.jsonContinuation + gateway.modules.services.serviceChat.mainServiceChat ---------------------------------------------------- [getChatDocumentsFromDocumentList] modules.datamodels.datamodelDocref +gateway.modules.services.serviceExtraction.mainServiceExtraction +---------------------------------------------------------------- + [extractContent] modules.interfaces.interfaceDbManagement + [extractContent] modules.shared.debugLogger + +gateway.modules.services.serviceExtraction.subPromptBuilderExtraction +--------------------------------------------------------------------- + [buildExtractionPrompt] modules.shared.debugLogger + +gateway.modules.services.serviceGeneration.mainServiceGeneration +---------------------------------------------------------------- + [getAdaptiveExtractionPrompt] modules.services.serviceExtraction.subPromptBuilderExtraction + [renderReport] modules.services.serviceGeneration.renderers.registry + [generateDocumentWithTwoPhases] modules.services.serviceGeneration.subContentGenerator + [generateDocumentWithTwoPhases] modules.services.serviceGeneration.subStructureGenerator + +gateway.modules.services.serviceGeneration.paths.codePath +--------------------------------------------------------- + [generateCode] modules.datamodels.datamodelDocument + [_getCodeRenderer] modules.services.serviceGeneration.renderers.registry + [_generateCodeStructure] modules.shared.jsonContinuation + [_generateSingleFileContent] modules.shared.jsonContinuation + +gateway.modules.services.serviceGeneration.renderers.rendererDocx +----------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererHtml +----------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererImage +------------------------------------------------------------------ + [_compressPromptWithAi] modules.datamodels.datamodelAi + [_generateAiImage] modules.datamodels.datamodelAi + +gateway.modules.services.serviceGeneration.renderers.rendererJson +----------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererMarkdown +--------------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererPdf +---------------------------------------------------------------- + [_getAiStylesWithPdfColors] modules.datamodels.datamodelAi + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererPptx +----------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererText +----------------------------------------------------------------- + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.renderers.rendererXlsx +----------------------------------------------------------------- + [_getAiStylesWithExcelColors] modules.datamodels.datamodelAi + [getAcceptedSectionTypes] modules.datamodels.datamodelJson + +gateway.modules.services.serviceGeneration.subContentGenerator +-------------------------------------------------------------- + [_generateImageSection] modules.datamodels.datamodelAi + [_generateSimpleSection] modules.datamodels.datamodelAi + [_generateSimpleSection] modules.shared.jsonUtils + +gateway.modules.services.serviceGeneration.subStructureGenerator +---------------------------------------------------------------- + [generateStructure] modules.datamodels.datamodelAi + gateway.modules.services.serviceUtils.mainServiceUtils ------------------------------------------------------ - [storeDebugMessageAndDocuments] modules.aichat.interfaceFeatureAiChat + [storeDebugMessageAndDocuments] modules.interfaces.interfaceDbChat [debugLogToFile] modules.shared.debugLogger [writeDebugArtifact] modules.shared.debugLogger [writeDebugFile] modules.shared.debugLogger @@ -409,9 +418,9 @@ gateway.modules.workflows.processing.modes.modeDynamic gateway.modules.workflows.processing.shared.placeholderFactory -------------------------------------------------------------- - [extractReviewContent] modules.aichat.datamodelFeatureAiChat - [extractLatestRefinementFeedback] modules.aichat.interfaceFeatureAiChat + [extractReviewContent] modules.datamodels.datamodelChat [extractLatestRefinementFeedback] modules.interfaces.interfaceDbApp + [extractLatestRefinementFeedback] modules.interfaces.interfaceDbChat gateway.modules.workflows.workflowManager ----------------------------------------- diff --git a/scripts/import_analysis.csv b/scripts/import_analysis.csv index 03015969..5fbe59c0 100644 --- a/scripts/import_analysis.csv +++ b/scripts/import_analysis.csv @@ -327,6 +327,9 @@ gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.attributeUtil gateway.modules.datamodels.datamodelWorkflowActions,modules.shared.frontendTypes,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,pydantic,header,Yes gateway.modules.datamodels.datamodelWorkflowActions,typing,header,Yes +gateway.modules.features.aichat.mainAiChat,logging,header,Yes +gateway.modules.features.aichat.mainAiChat,modules.aicore.aicoreModelRegistry,function onStart,Yes +gateway.modules.features.aichat.mainAiChat,typing,header,Yes gateway.modules.features.automation.datamodelFeatureAutomation,modules.shared.attributeUtils,header,Yes gateway.modules.features.automation.datamodelFeatureAutomation,pydantic,header,Yes gateway.modules.features.automation.datamodelFeatureAutomation,typing,header,Yes From a0b9a6e4b58d62c24cdcce36a54863729c0dcef9 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 24 Jan 2026 00:42:19 +0100 Subject: [PATCH 18/32] reference fixes --- modules/aicore/aicorePluginPerplexity.py | 2 +- modules/datamodels/datamodelChat.py | 36 +++++++++---------- modules/datamodels/datamodelFiles.py | 4 +-- modules/interfaces/interfaceDbManagement.py | 5 +-- modules/interfaces/interfaceRbac.py | 1 - modules/routes/routeDataConnections.py | 1 + modules/routes/routeDataFiles.py | 10 +++--- modules/routes/routeDataUsers.py | 2 +- modules/routes/routeSecurityGoogle.py | 1 + modules/routes/routeSecurityMsft.py | 1 + .../serviceAi/subContentExtraction.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../workflows/processing/core/taskPlanner.py | 2 +- .../workflows/processing/modes/modeDynamic.py | 2 +- .../workflows/processing/workflowProcessor.py | 2 +- 15 files changed, 37 insertions(+), 36 deletions(-) diff --git a/modules/aicore/aicorePluginPerplexity.py b/modules/aicore/aicorePluginPerplexity.py index e751f1e9..e6d1ba10 100644 --- a/modules/aicore/aicorePluginPerplexity.py +++ b/modules/aicore/aicorePluginPerplexity.py @@ -6,7 +6,7 @@ from typing import List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from .aicoreBase import BaseConnectorAi -from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings, AiCallPromptWebSearch, AiCallPromptWebCrawl +from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings, AiCallPromptWebSearch, AiCallPromptWebCrawl, AiCallOptions from modules.datamodels.datamodelTools import CountryCodes # Configure logger diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index c2838ad3..328bee22 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -14,11 +14,11 @@ class ChatStat(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: str = Field( - description="ID of the mandate this stat belongs to" + mandateId: Optional[str] = Field( + default="", description="ID of the mandate this stat belongs to" ) - featureInstanceId: str = Field( - description="ID of the feature instance this stat belongs to" + featureInstanceId: Optional[str] = Field( + default="", description="ID of the feature instance this stat belongs to" ) workflowId: Optional[str] = Field( None, description="Foreign key to workflow (for workflow stats)" @@ -57,11 +57,11 @@ class ChatLog(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: str = Field( - description="ID of the mandate this log belongs to" + mandateId: Optional[str] = Field( + default="", description="ID of the mandate this log belongs to" ) - featureInstanceId: str = Field( - description="ID of the feature instance this log belongs to" + featureInstanceId: Optional[str] = Field( + default="", description="ID of the feature instance this log belongs to" ) workflowId: str = Field(description="Foreign key to workflow") message: str = Field(description="Log message") @@ -110,11 +110,11 @@ class ChatDocument(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: str = Field( - description="ID of the mandate this document belongs to" + mandateId: Optional[str] = Field( + default="", description="ID of the mandate this document belongs to" ) - featureInstanceId: str = Field( - description="ID of the feature instance this document belongs to" + featureInstanceId: Optional[str] = Field( + default="", description="ID of the feature instance this document belongs to" ) messageId: str = Field(description="Foreign key to message") fileId: str = Field(description="Foreign key to file") @@ -224,11 +224,11 @@ class ChatMessage(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: str = Field( - description="ID of the mandate this message belongs to" + mandateId: Optional[str] = Field( + default="", description="ID of the mandate this message belongs to" ) - featureInstanceId: str = Field( - description="ID of the feature instance this message belongs to" + featureInstanceId: Optional[str] = Field( + default="", description="ID of the feature instance this message belongs to" ) workflowId: str = Field(description="Foreign key to workflow") parentMessageId: Optional[str] = Field( @@ -327,8 +327,8 @@ registerModelLabels( class ChatWorkflow(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: str = Field(description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: str = Field(description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: Optional[str] = Field(default="", description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: Optional[str] = Field(default="", description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index 07880f3d..f1b07eb3 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -12,8 +12,8 @@ import base64 class FileItem(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: str = Field(description="ID of the mandate this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: str = Field(description="ID of the feature instance this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + mandateId: Optional[str] = Field(default="", description="ID of the mandate this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: Optional[str] = Field(default="", description="ID of the feature instance this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileName: str = Field(description="Name of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) mimeType: str = Field(description="MIME type of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileHash: str = Field(description="Hash of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index b514cbf3..a26e8c98 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -989,8 +989,9 @@ class ComponentObjects: fileHash = hashlib.sha256(content).hexdigest() # Use mandateId and featureInstanceId from context for proper data isolation - mandateId = self.mandateId - featureInstanceId = self.featureInstanceId + # Convert None to empty string to satisfy Pydantic validation + mandateId = self.mandateId or "" + featureInstanceId = self.featureInstanceId or "" # Create FileItem instance fileItem = FileItem( diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 6b432a2b..a99bfd0b 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -62,7 +62,6 @@ def getRecordsetWithRBAC( # SysAdmin bypass: SysAdmin users have full access to all tables isSysAdmin = getattr(currentUser, 'isSysAdmin', False) if isSysAdmin: - logger.debug(f"SysAdmin user {currentUser.id} bypassing RBAC for table {table}") # Direct access without RBAC filtering # Note: getRecordset doesn't support orderBy/limit - these are only used in RBAC path return connector.getRecordset(modelClass, recordFilter=recordFilter) diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 210e0522..2dade569 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -20,6 +20,7 @@ import math from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority, ConnectionStatus from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter +from modules.auth.tokenRefreshService import token_refresh_service from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.interfaces.interfaceDbApp import getInterface from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 66345ce5..1a84b7e4 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -136,15 +136,13 @@ async def upload_file( else: # new_file message = "File uploaded successfully" - # If workflowId is provided, update the file information - if workflowId: - updateData = {"workflowId": workflowId} - managementInterface.updateFile(fileItem.id, updateData) - fileItem.workflowId = workflowId - # Convert FileItem to dictionary for JSON response fileMeta = fileItem.model_dump() + # If workflowId is provided, include it in the response (not stored in FileItem model) + if workflowId: + fileMeta["workflowId"] = workflowId + # Response with duplicate information return JSONResponse({ "message": message, diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index ed55b11f..b3da0c2e 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -21,7 +21,7 @@ import modules.interfaces.interfaceDbApp as interfaceDbApp from modules.auth import limiter, getRequestContext, RequestContext # Import the attribute definition and helper functions -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict # Configure logger diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 39edcf8b..a4795243 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -17,6 +17,7 @@ from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection from modules.auth import getCurrentUser, limiter from modules.auth import createAccessToken, setAccessTokenCookie, createRefreshToken, setRefreshTokenCookie +from modules.auth.tokenManager import TokenManager from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp # Configure logger diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index bc637222..921ddafe 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -18,6 +18,7 @@ from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatu from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter from modules.auth import createAccessToken, setAccessTokenCookie, createRefreshToken, setRefreshTokenCookie +from modules.auth.tokenManager import TokenManager from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp # Configure logger diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py index 696ba377..a7250a3a 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -15,7 +15,7 @@ import base64 from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ChatDocument -from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent +from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent, ExtractionOptions, MergeStrategy from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index cdaf55b6..1ab4a7ee 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -6,7 +6,7 @@ import time import json from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ActionResult, ActionDocument -from modules.datamodels.datamodelAi import AiCallOptions +from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 94c695cb..b1e1def7 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, WorkflowModeEnum from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index 45b92961..1510e512 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from typing import List, Dict, Any from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, - ActionResult, Observation, ObservationPreview, ReviewResult + ActionResult, Observation, ObservationPreview, ReviewResult, ReviewContext ) from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index c73d280b..11879e9d 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -13,7 +13,7 @@ from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped -from modules.datamodels.datamodelAi import OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.datamodels.datamodelAi import OperationTypeEnum, PriorityEnum, ProcessingModeEnum, AiCallOptions from modules.shared.jsonUtils import extractJsonString, repairBrokenJson if TYPE_CHECKING: From 50e3fce12b4f682ee4864224c0e9748e3c4ec7cf Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 24 Jan 2026 02:06:49 +0100 Subject: [PATCH 19/32] fixed automation and trustee --- app.py | 3 + .../automation/routeFeatureAutomation.py | 30 +-- modules/features/featureRegistry.py | 10 + modules/features/trustee/mainTrustee.py | 216 +++++++++++++++++- modules/interfaces/interfaceDbChat.py | 25 +- modules/routes/routeAdminFeatures.py | 15 +- modules/shared/attributeUtils.py | 4 +- modules/workflows/automation/mainWorkflow.py | 28 ++- 8 files changed, 296 insertions(+), 35 deletions(-) diff --git a/app.py b/app.py index 4e3e0c67..57274338 100644 --- a/app.py +++ b/app.py @@ -47,6 +47,9 @@ class DailyRotatingFileHandler(RotatingFileHandler): def _updateFileIfNeeded(self): """Update the log file if the date has changed""" + # Guard against interpreter shutdown when datetime may be None + if datetime is None: + return False today = datetime.now().strftime("%Y%m%d") if self.currentDate != today: diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 49c1606e..4eef9381 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -14,7 +14,7 @@ import json # Import interfaces and models from modules.interfaces.interfaceDbChat import getInterface as getChatInterface -from modules.auth import getCurrentUser, limiter +from modules.auth import getCurrentUser, limiter, getRequestContext, RequestContext from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition from modules.datamodels.datamodelChat import ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict @@ -46,7 +46,7 @@ router = APIRouter( async def get_automations( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[AutomationDefinition]: """ Get automation definitions with optional pagination, sorting, and filtering. @@ -69,7 +69,7 @@ async def get_automations( detail=f"Invalid pagination parameter: {str(e)}" ) - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) result = chatInterface.getAllAutomationDefinitions(pagination=paginationParams) # If pagination was requested, result is PaginatedResult @@ -111,11 +111,11 @@ async def get_automations( async def create_automation( request: Request, automation: AutomationDefinition, - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> AutomationDefinition: """Create a new automation definition""" try: - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) automationData = automation.model_dump() created = chatInterface.createAutomationDefinition(automationData) return created @@ -132,7 +132,7 @@ async def create_automation( @limiter.limit("30/minute") async def get_automation_templates( request: Request, - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> JSONResponse: """ Get automation templates from backend module. @@ -160,11 +160,11 @@ async def get_automation_attributes( async def get_automation( request: Request, automationId: str = Path(..., description="Automation ID"), - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> AutomationDefinition: """Get a single automation definition by ID""" try: - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) automation = chatInterface.getAutomationDefinition(automationId) if not automation: raise HTTPException( @@ -188,11 +188,11 @@ async def update_automation( request: Request, automationId: str = Path(..., description="Automation ID"), automation: AutomationDefinition = Body(...), - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> AutomationDefinition: """Update an automation definition""" try: - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) automationData = automation.model_dump() updated = chatInterface.updateAutomationDefinition(automationId, automationData) return updated @@ -215,11 +215,11 @@ async def update_automation( async def delete_automation( request: Request, automationId: str = Path(..., description="Automation ID"), - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> Response: """Delete an automation definition""" try: - chatInterface = getChatInterface(currentUser) + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) success = chatInterface.deleteAutomationDefinition(automationId) if success: return Response(status_code=204) @@ -244,15 +244,15 @@ async def delete_automation( @router.post("/{automationId}/execute", response_model=ChatWorkflow) @limiter.limit("5/minute") -async def execute_automation( +async def execute_automation_route( request: Request, automationId: str = Path(..., description="Automation ID"), - currentUser = Depends(getCurrentUser) + context: RequestContext = Depends(getRequestContext) ) -> ChatWorkflow: """Execute an automation immediately (test mode)""" try: from modules.services import getInterface as getServices - services = getServices(currentUser, None) + services = getServices(context.user, context.mandateId) workflow = await executeAutomation(automationId, services) return workflow except HTTPException: diff --git a/modules/features/featureRegistry.py b/modules/features/featureRegistry.py index 29eb35c9..4bf6d82e 100644 --- a/modules/features/featureRegistry.py +++ b/modules/features/featureRegistry.py @@ -37,6 +37,7 @@ def discoverFeatureContainers() -> List[str]: def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: """ Dynamically load and register routers from all discovered feature containers. + Also registers feature template roles and AccessRules in the database. """ results = {} pattern = os.path.join(FEATURES_DIR, "*", "routeFeature*.py") @@ -64,6 +65,15 @@ def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: logger.error(f"Failed to load router from {featureDir}: {e}") results[featureDir] = {"status": "error", "error": str(e)} + # Register features in RBAC catalog and sync template roles to database + from modules.security.rbacCatalog import getCatalogService + catalogService = getCatalogService() + registrationResults = registerAllFeaturesInCatalog(catalogService) + + for featureName, success in registrationResults.items(): + if featureName in results: + results[featureName]["rbac_registered"] = success + return results diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 176317a7..8cda0cd0 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -103,7 +103,8 @@ RESOURCE_OBJECTS = [ }, ] -# Template roles for this feature +# Template roles for this feature with AccessRules +# Each role defines default UI and DATA permissions TEMPLATE_ROLES = [ { "roleLabel": "trustee-admin", @@ -111,7 +112,13 @@ TEMPLATE_ROLES = [ "en": "Trustee Administrator - Full access to all trustee data and settings", "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" - } + }, + "accessRules": [ + # Full UI access + {"context": "UI", "item": None, "view": True}, + # Full DATA access + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + ] }, { "roleLabel": "trustee-accountant", @@ -119,7 +126,13 @@ TEMPLATE_ROLES = [ "en": "Trustee Accountant - Manage accounting and financial data", "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", "fr": "Comptable fiduciaire - Gérer les données comptables et financières" - } + }, + "accessRules": [ + # Full UI access + {"context": "UI", "item": None, "view": True}, + # Group-level DATA access + {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, + ] }, { "roleLabel": "trustee-client", @@ -127,7 +140,13 @@ TEMPLATE_ROLES = [ "en": "Trustee Client - View own accounting data and documents", "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" - } + }, + "accessRules": [ + # Full UI access + {"context": "UI", "item": None, "view": True}, + # Own records only (MY level) + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + ] }, ] @@ -185,9 +204,198 @@ def registerFeature(catalogService) -> bool: meta=resObj.get("meta") ) + # Sync template roles to database (with AccessRules) + _syncTemplateRolesToDb() + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") return True except Exception as e: logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") return False + + +def _syncTemplateRolesToDb() -> int: + """ + Sync template roles and their AccessRules to the database. + Creates global template roles (mandateId=None) if they don't exist. + + Returns: + Number of roles created/updated + """ + try: + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + rootInterface = getRootInterface() + db = rootInterface.db + + # Get existing template roles for this feature + existingRoles = db.getRecordset( + Role, + recordFilter={"featureCode": FEATURE_CODE, "mandateId": None} + ) + existingRoleLabels = {r.get("roleLabel"): r.get("id") for r in existingRoles} + + createdCount = 0 + for roleTemplate in TEMPLATE_ROLES: + roleLabel = roleTemplate["roleLabel"] + + if roleLabel in existingRoleLabels: + roleId = existingRoleLabels[roleLabel] + logger.debug(f"Template role '{roleLabel}' already exists with ID {roleId}") + + # Ensure AccessRules exist for this role + _ensureAccessRulesForRole(db, roleId, roleTemplate.get("accessRules", [])) + else: + # Create new template role + newRole = Role( + roleLabel=roleLabel, + description=roleTemplate.get("description", {}), + featureCode=FEATURE_CODE, + mandateId=None, # Global template + featureInstanceId=None, + isSystemRole=False + ) + createdRole = db.recordCreate(Role, newRole.model_dump()) + roleId = createdRole.get("id") + + # Create AccessRules for this role + _ensureAccessRulesForRole(db, roleId, roleTemplate.get("accessRules", [])) + + logger.info(f"Created template role '{roleLabel}' with ID {roleId}") + createdCount += 1 + + if createdCount > 0: + logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") + + # Repair instance-specific roles that are missing AccessRules + _repairInstanceRolesAccessRules(db, existingRoleLabels) + + return createdCount + + except Exception as e: + logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") + return 0 + + +def _repairInstanceRolesAccessRules(db, templateRoleLabels: Dict[str, str]) -> int: + """ + Repair instance-specific roles by copying AccessRules from their template roles. + This ensures instance roles created before AccessRules were defined get updated. + + Args: + db: Database connector + templateRoleLabels: Dict mapping roleLabel to template role ID + + Returns: + Number of instance roles repaired + """ + from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + repairedCount = 0 + + # Get all instance-specific roles for this feature (mandateId is NOT None) + allRoles = db.getRecordset(Role, recordFilter={"featureCode": FEATURE_CODE}) + instanceRoles = [r for r in allRoles if r.get("mandateId") is not None] + + for instanceRole in instanceRoles: + roleLabel = instanceRole.get("roleLabel") + instanceRoleId = instanceRole.get("id") + + # Find matching template role + templateRoleId = templateRoleLabels.get(roleLabel) + if not templateRoleId: + continue + + # Check if instance role has AccessRules + existingRules = db.getRecordset(AccessRule, recordFilter={"roleId": instanceRoleId}) + if existingRules: + continue # Already has rules, skip + + # Copy AccessRules from template role + templateRules = db.getRecordset(AccessRule, recordFilter={"roleId": templateRoleId}) + if not templateRules: + continue # Template has no rules + + for rule in templateRules: + newRule = AccessRule( + roleId=instanceRoleId, + context=rule.get("context"), + item=rule.get("item"), + view=rule.get("view", False), + read=rule.get("read"), + create=rule.get("create"), + update=rule.get("update"), + delete=rule.get("delete"), + ) + db.recordCreate(AccessRule, newRule.model_dump()) + + logger.info(f"Repaired instance role '{roleLabel}' (ID: {instanceRoleId}): copied {len(templateRules)} AccessRules from template") + repairedCount += 1 + + if repairedCount > 0: + logger.info(f"Feature '{FEATURE_CODE}': Repaired {repairedCount} instance roles with missing AccessRules") + + return repairedCount + + +def _ensureAccessRulesForRole(db, roleId: str, ruleTemplates: List[Dict[str, Any]]) -> int: + """ + Ensure AccessRules exist for a role based on templates. + + Args: + db: Database connector + roleId: Role ID + ruleTemplates: List of rule templates + + Returns: + Number of rules created + """ + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext + + # Get existing rules for this role + existingRules = db.getRecordset(AccessRule, recordFilter={"roleId": roleId}) + + # Create a set of existing rule signatures to avoid duplicates + existingSignatures = set() + for rule in existingRules: + sig = (rule.get("context"), rule.get("item")) + existingSignatures.add(sig) + + createdCount = 0 + for template in ruleTemplates: + context = template.get("context", "UI") + item = template.get("item") + sig = (context, item) + + if sig in existingSignatures: + continue + + # Map context string to enum + if context == "UI": + contextEnum = AccessRuleContext.UI + elif context == "DATA": + contextEnum = AccessRuleContext.DATA + elif context == "RESOURCE": + contextEnum = AccessRuleContext.RESOURCE + else: + contextEnum = context + + newRule = AccessRule( + roleId=roleId, + context=contextEnum, + item=item, + view=template.get("view", False), + read=template.get("read"), + create=template.get("create"), + update=template.get("update"), + delete=template.get("delete"), + ) + db.recordCreate(AccessRule, newRule.model_dump()) + createdCount += 1 + + if createdCount > 0: + logger.debug(f"Created {createdCount} AccessRules for role {roleId}") + + return createdCount diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index bbe2d9ce..de16d6af 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -1729,8 +1729,14 @@ class ChatObjects: totalPages=totalPages ) - def getAutomationDefinition(self, automationId: str) -> Optional[AutomationDefinition]: - """Returns an automation definition by ID if user has access, with computed status.""" + def getAutomationDefinition(self, automationId: str, includeSystemFields: bool = False) -> Optional[AutomationDefinition]: + """Returns an automation definition by ID if user has access, with computed status. + + Args: + automationId: ID of the automation to get + includeSystemFields: If True, returns raw dict with system fields (_createdBy, etc). + If False (default), returns Pydantic model without system fields. + """ try: # Use RBAC filtering filtered = getRecordsetWithRBAC(self.db, @@ -1749,6 +1755,16 @@ class ChatObjects: automation["executionLogs"] = [] # Enrich with user and mandate names self._enrichAutomationWithUserAndMandate(automation) + + # For internal use (execution), return raw dict with system fields + if includeSystemFields: + # Return as simple namespace object so getattr works + class AutomationWithSystemFields: + def __init__(self, data): + for key, value in data.items(): + setattr(self, key, value) + return AutomationWithSystemFields(automation) + # Clean metadata fields and return Pydantic model cleanedRecord = {k: v for k, v in automation.items() if not k.startswith("_")} return AutomationDefinition(**cleanedRecord) @@ -1771,9 +1787,12 @@ class ChatObjects: # Ensure database connector has correct userId context # The connector should have been initialized with userId, but ensure it's updated - if self.userId and hasattr(self.db, 'updateContext'): + if not self.userId: + logger.error(f"createAutomationDefinition: userId is not set! Cannot set _createdBy. currentUser={self.currentUser}") + elif hasattr(self.db, 'updateContext'): try: self.db.updateContext(self.userId) + logger.debug(f"createAutomationDefinition: Updated database context with userId={self.userId}") except Exception as e: logger.warning(f"Could not update database context: {e}") diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 575af01e..9cb023c6 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -248,7 +248,10 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict recordFilter={"userId": userId, "featureInstanceId": instanceId} ) + logger.debug(f"_getInstancePermissions: userId={userId}, instanceId={instanceId}, featureAccesses={len(featureAccesses) if featureAccesses else 0}") + if not featureAccesses: + logger.debug(f"_getInstancePermissions: No FeatureAccess found for user {userId} and instance {instanceId}") return permissions # Get role IDs via FeatureAccessRole junction table @@ -259,7 +262,10 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict ) roleIds = [far.get("roleId") for far in featureAccessRoles] + logger.debug(f"_getInstancePermissions: featureAccessId={featureAccessId}, roleIds={roleIds}") + if not roleIds: + logger.debug(f"_getInstancePermissions: No roles found for FeatureAccess {featureAccessId}") return permissions # Get permissions (AccessRules) for all roles @@ -269,6 +275,8 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict recordFilter={"roleId": roleId} ) + logger.debug(f"_getInstancePermissions: roleId={roleId}, accessRules={len(accessRules) if accessRules else 0}") + for rule in accessRules: context = rule.get("context", "") item = rule.get("item", "") @@ -303,8 +311,13 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict # Handle UI context (views) elif context == "UI" or context == AccessRuleContext.UI: + ruleView = rule.get("view", False) if item: - permissions["views"][item] = permissions["views"].get(item, False) or rule.get("view", False) + # Specific view rule + permissions["views"][item] = permissions["views"].get(item, False) or ruleView + elif ruleView: + # item=None means all views - set a wildcard flag + permissions["views"]["_all"] = True return permissions diff --git a/modules/shared/attributeUtils.py b/modules/shared/attributeUtils.py index 5f6c2531..863d7f36 100644 --- a/modules/shared/attributeUtils.py +++ b/modules/shared/attributeUtils.py @@ -5,6 +5,7 @@ Shared utilities for model attributes and labels. """ from pydantic import BaseModel, Field, ConfigDict +from pydantic_core import PydanticUndefined from typing import Dict, Any, List, Type, Optional, Union import inspect import importlib @@ -212,7 +213,8 @@ def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguag # Don't call it here - it's meant to be called per-instance # Instead, store a marker that indicates it exists field_default = None # Frontend should use first option or specific logic - elif default_value is not ...: # Ellipsis means no default + elif default_value is not ... and default_value is not PydanticUndefined: + # Ellipsis or PydanticUndefined means no default # Convert enum to its value if it's an enum if hasattr(default_value, 'value'): field_default = default_value.value diff --git a/modules/workflows/automation/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py index 23bbf125..503d1d13 100644 --- a/modules/workflows/automation/mainWorkflow.py +++ b/modules/workflows/automation/mainWorkflow.py @@ -76,8 +76,8 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: } try: - # 1. Load automation definition - automation = services.interfaceDbChat.getAutomationDefinition(automationId) + # 1. Load automation definition (with system fields for _createdBy access) + automation = services.interfaceDbChat.getAutomationDefinition(automationId, includeSystemFields=True) if not automation: raise ValueError(f"Automation {automationId} not found") @@ -105,18 +105,17 @@ async def executeAutomation(automationId: str, services) -> ChatWorkflow: # 3. Get user who created automation creatorUserId = getattr(automation, "_createdBy", None) - # CRITICAL: Automation MUST run as creator user only, or fail + # _createdBy is a system attribute - must be present if not creatorUserId: errorMsg = f"Automation {automationId} has no creator user (_createdBy field missing). Cannot execute automation." logger.error(errorMsg) executionLog["messages"].append(errorMsg) raise ValueError(errorMsg) - # Get user from database using services + # Get creator user from database creatorUser = services.interfaceDbApp.getUser(creatorUserId) if not creatorUser: raise ValueError(f"Creator user {creatorUserId} not found") - executionLog["messages"].append(f"Using creator user: {creatorUserId}") # 4. Create UserInputRequest from plan @@ -205,10 +204,17 @@ async def syncAutomationEvents(services, eventUser) -> Dict[str, Any]: registeredEvents = {} for automation in filtered: - automationId = automation.id - isActive = automation.active if hasattr(automation, 'active') else False - currentEventId = automation.eventId if hasattr(automation, 'eventId') else None - schedule = automation.schedule if hasattr(automation, 'schedule') else None + # Handle both dict and object access patterns + if isinstance(automation, dict): + automationId = automation.get('id') + isActive = automation.get('active', False) + currentEventId = automation.get('eventId') + schedule = automation.get('schedule') + else: + automationId = automation.id + isActive = automation.active if hasattr(automation, 'active') else False + currentEventId = automation.eventId if hasattr(automation, 'eventId') else None + schedule = automation.schedule if hasattr(automation, 'schedule') else None if not schedule: logger.warning(f"Automation {automationId} has no schedule, skipping") @@ -287,8 +293,8 @@ def createAutomationEventHandler(automationId: str, eventUser): # Get services for event user (provides access to interfaces) eventServices = getServices(eventUser, None) - # Load automation using event user context - automation = eventServices.interfaceDbChat.getAutomationDefinition(automationId) + # Load automation using event user context (with system fields for _createdBy access) + automation = eventServices.interfaceDbChat.getAutomationDefinition(automationId, includeSystemFields=True) if not automation or not getattr(automation, "active", False): logger.warning(f"Automation {automationId} not found or not active, skipping execution") return From efc28879c39ca2e98107688343c4699b647d8157 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 24 Jan 2026 09:43:46 +0100 Subject: [PATCH 20/32] access rules editor enhanced --- modules/features/trustee/mainTrustee.py | 100 +++--- .../features/trustee/routeFeatureTrustee.py | 328 +++++++++++++++++- modules/routes/routeAdminRbacRules.py | 44 +++ 3 files changed, 415 insertions(+), 57 deletions(-) diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 8cda0cd0..5ee0ed08 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -16,76 +16,48 @@ FEATURE_LABEL = {"en": "Trustee", "de": "Treuhand", "fr": "Fiduciaire"} FEATURE_ICON = "mdi-briefcase" # UI Objects for RBAC catalog +# Note: organisations and contracts removed - feature instance = organisation UI_OBJECTS = [ { - "objectKey": "ui.feature.trustee.organisations", - "label": {"en": "Organisations", "de": "Organisationen", "fr": "Organisations"}, - "meta": {"area": "organisations"} + "objectKey": "ui.feature.trustee.dashboard", + "label": {"en": "Dashboard", "de": "Dashboard", "fr": "Tableau de bord"}, + "meta": {"area": "dashboard"} }, { - "objectKey": "ui.feature.trustee.contracts", - "label": {"en": "Contracts", "de": "Verträge", "fr": "Contrats"}, - "meta": {"area": "contracts"} + "objectKey": "ui.feature.trustee.positions", + "label": {"en": "Positions", "de": "Positionen", "fr": "Positions"}, + "meta": {"area": "positions"} }, { - "objectKey": "ui.feature.trustee.contracts.tab.documents", - "label": {"en": "Contract Documents", "de": "Vertragsdokumente", "fr": "Documents contractuels"}, - "meta": {"area": "contracts", "element": "tab.documents"} + "objectKey": "ui.feature.trustee.documents", + "label": {"en": "Documents", "de": "Dokumente", "fr": "Documents"}, + "meta": {"area": "documents"} }, { - "objectKey": "ui.feature.trustee.contracts.tab.positions", - "label": {"en": "Contract Positions", "de": "Vertragspositionen", "fr": "Positions contractuelles"}, - "meta": {"area": "contracts", "element": "tab.positions"} + "objectKey": "ui.feature.trustee.position-documents", + "label": {"en": "Position Documents", "de": "Positions-Dokumente", "fr": "Documents de position"}, + "meta": {"area": "position-documents"} }, { - "objectKey": "ui.feature.trustee.access", - "label": {"en": "Access Management", "de": "Zugriffsverwaltung", "fr": "Gestion des accès"}, - "meta": {"area": "access"} - }, - { - "objectKey": "ui.feature.trustee.roles", - "label": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, - "meta": {"area": "roles"} + "objectKey": "ui.feature.trustee.instance-roles", + "label": {"en": "Instance Roles & Permissions", "de": "Instanz-Rollen & Berechtigungen", "fr": "Rôles et permissions d'instance"}, + "meta": {"area": "admin", "admin_only": True} }, ] # Resource Objects for RBAC catalog +# Note: organisations and contracts removed - feature instance = organisation RESOURCE_OBJECTS = [ - { - "objectKey": "resource.feature.trustee.organisations.create", - "label": {"en": "Create Organisation", "de": "Organisation erstellen", "fr": "Créer organisation"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/organisations", "method": "POST"} - }, - { - "objectKey": "resource.feature.trustee.organisations.update", - "label": {"en": "Update Organisation", "de": "Organisation aktualisieren", "fr": "Modifier organisation"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/organisations/{orgId}", "method": "PUT"} - }, - { - "objectKey": "resource.feature.trustee.organisations.delete", - "label": {"en": "Delete Organisation", "de": "Organisation löschen", "fr": "Supprimer organisation"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/organisations/{orgId}", "method": "DELETE"} - }, - { - "objectKey": "resource.feature.trustee.contracts.create", - "label": {"en": "Create Contract", "de": "Vertrag erstellen", "fr": "Créer contrat"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/contracts", "method": "POST"} - }, - { - "objectKey": "resource.feature.trustee.contracts.update", - "label": {"en": "Update Contract", "de": "Vertrag aktualisieren", "fr": "Modifier contrat"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/contracts/{contractId}", "method": "PUT"} - }, - { - "objectKey": "resource.feature.trustee.contracts.delete", - "label": {"en": "Delete Contract", "de": "Vertrag löschen", "fr": "Supprimer contrat"}, - "meta": {"endpoint": "/api/trustee/{instanceId}/contracts/{contractId}", "method": "DELETE"} - }, { "objectKey": "resource.feature.trustee.documents.create", "label": {"en": "Upload Document", "de": "Dokument hochladen", "fr": "Télécharger document"}, "meta": {"endpoint": "/api/trustee/{instanceId}/documents", "method": "POST"} }, + { + "objectKey": "resource.feature.trustee.documents.update", + "label": {"en": "Update Document", "de": "Dokument aktualisieren", "fr": "Modifier document"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/documents/{documentId}", "method": "PUT"} + }, { "objectKey": "resource.feature.trustee.documents.delete", "label": {"en": "Delete Document", "de": "Dokument löschen", "fr": "Supprimer document"}, @@ -96,15 +68,26 @@ RESOURCE_OBJECTS = [ "label": {"en": "Create Position", "de": "Position erstellen", "fr": "Créer position"}, "meta": {"endpoint": "/api/trustee/{instanceId}/positions", "method": "POST"} }, + { + "objectKey": "resource.feature.trustee.positions.update", + "label": {"en": "Update Position", "de": "Position aktualisieren", "fr": "Modifier position"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/positions/{positionId}", "method": "PUT"} + }, { "objectKey": "resource.feature.trustee.positions.delete", "label": {"en": "Delete Position", "de": "Position löschen", "fr": "Supprimer position"}, "meta": {"endpoint": "/api/trustee/{instanceId}/positions/{positionId}", "method": "DELETE"} }, + { + "objectKey": "resource.feature.trustee.instance-roles.manage", + "label": {"en": "Manage Instance Roles", "de": "Instanz-Rollen verwalten", "fr": "Gérer les rôles d'instance"}, + "meta": {"endpoint": "/api/trustee/{instanceId}/instance-roles", "method": "ALL", "admin_only": True} + }, ] # Template roles for this feature with AccessRules # Each role defines default UI and DATA permissions +# Note: UI item=None means ALL views, specific items restrict to named views TEMPLATE_ROLES = [ { "roleLabel": "trustee-admin", @@ -114,10 +97,12 @@ TEMPLATE_ROLES = [ "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" }, "accessRules": [ - # Full UI access + # Full UI access (all views including admin views) {"context": "UI", "item": None, "view": True}, # Full DATA access {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + # Admin resource: manage instance roles + {"context": "RESOURCE", "item": "instance-roles.manage", "view": True}, ] }, { @@ -128,8 +113,11 @@ TEMPLATE_ROLES = [ "fr": "Comptable fiduciaire - Gérer les données comptables et financières" }, "accessRules": [ - # Full UI access - {"context": "UI", "item": None, "view": True}, + # UI access to main views (not admin views) + {"context": "UI", "item": "dashboard", "view": True}, + {"context": "UI", "item": "positions", "view": True}, + {"context": "UI", "item": "documents", "view": True}, + {"context": "UI", "item": "position-documents", "view": True}, # Group-level DATA access {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, ] @@ -142,8 +130,10 @@ TEMPLATE_ROLES = [ "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" }, "accessRules": [ - # Full UI access - {"context": "UI", "item": None, "view": True}, + # UI access to main views only (read-only focus) + {"context": "UI", "item": "dashboard", "view": True}, + {"context": "UI", "item": "positions", "view": True}, + {"context": "UI", "item": "documents", "view": True}, # Own records only (MY level) {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, ] diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index e05f6dc0..9e30951a 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -115,6 +115,66 @@ async def _validateInstanceAccess(instanceId: str, context: RequestContext) -> s return str(instance.mandateId) +# ============================================================================ +# ATTRIBUTES ENDPOINT (for FormGeneratorTable) +# ============================================================================ + +# Mapping of entity names to Pydantic model classes +_TRUSTEE_ENTITY_MODELS = { + "TrusteeOrganisation": TrusteeOrganisation, + "TrusteeRole": TrusteeRole, + "TrusteeAccess": TrusteeAccess, + "TrusteeContract": TrusteeContract, + "TrusteeDocument": TrusteeDocument, + "TrusteePosition": TrusteePosition, + "TrusteePositionDocument": TrusteePositionDocument, +} + + +@router.get("/{instanceId}/attributes/{entityType}") +@limiter.limit("30/minute") +async def getEntityAttributes( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + entityType: str = Path(..., description="Entity type (e.g., TrusteeDocument)"), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Get attribute definitions for a Trustee entity. + Used by FormGeneratorTable for dynamic column generation. + """ + # Validate instance access + await _validateInstanceAccess(instanceId, context) + + # Check if entity type is valid + if entityType not in _TRUSTEE_ENTITY_MODELS: + raise HTTPException( + status_code=404, + detail=f"Unknown entity type: {entityType}. Valid types: {list(_TRUSTEE_ENTITY_MODELS.keys())}" + ) + + # Get the model class + modelClass = _TRUSTEE_ENTITY_MODELS[entityType] + + # Import the attribute utils + from modules.shared.attributeUtils import getModelAttributeDefinitions + + try: + attrDefs = getModelAttributeDefinitions(modelClass) + # Filter to only visible attributes + visibleAttrs = [ + attr for attr in attrDefs.get("attributes", []) + if isinstance(attr, dict) and attr.get("visible", True) + ] + return {"attributes": visibleAttrs} + except Exception as e: + logger.error(f"Error getting attributes for {entityType}: {e}") + raise HTTPException( + status_code=500, + detail=f"Error getting attributes for {entityType}: {str(e)}" + ) + + # ============================================================================ # OPTIONS ENDPOINTS (for dropdowns) # ============================================================================ @@ -131,7 +191,7 @@ async def getOrganisationOptions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllOrganisations(None) items = result.items if hasattr(result, 'items') else result - return [{"value": org.id, "label": org.label or org.id} for org in items] + return [{"value": org["id"], "label": org.get("label") or org["id"]} for org in items] @router.get("/{instanceId}/roles/options", response_model=List[Dict[str, Any]]) @@ -146,7 +206,7 @@ async def getRoleOptions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllRoles(None) items = result.items if hasattr(result, 'items') else result - return [{"value": role.id, "label": role.desc or role.id} for role in items] + return [{"value": role["id"], "label": role.get("desc") or role["id"]} for role in items] @router.get("/{instanceId}/contracts/options", response_model=List[Dict[str, Any]]) @@ -1136,3 +1196,267 @@ async def deletePositionDocument( if not success: raise HTTPException(status_code=400, detail="Failed to delete link") return {"message": f"Link {linkId} deleted"} + + +# ===== Instance Roles Management ===== +# These endpoints allow feature admins to manage instance-specific roles and their AccessRules + +from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + +async def _validateInstanceAdmin(instanceId: str, context: RequestContext) -> str: + """ + Validate that the user has admin access to the feature instance. + Returns the mandateId if authorized. + + This checks for the RESOURCE permission 'instance-roles.manage'. + """ + mandateId = await _validateInstanceAccess(instanceId, context) + + # SysAdmin always has access + if context.user.isSysAdmin: + return mandateId + + # Check for instance-roles.manage resource permission + featureInterface = getFeatureInterface() + permissions = featureInterface.getUserPermissionsForInstance(context.user.id, instanceId) + + if not permissions: + raise HTTPException( + status_code=403, + detail="Keine Berechtigung zur Rollenverwaltung" + ) + + # Check for resource permission + resourcePermissions = permissions.get("resources", {}) + if not resourcePermissions.get("instance-roles.manage"): + raise HTTPException( + status_code=403, + detail="Keine Berechtigung zur Rollenverwaltung" + ) + + return mandateId + + +@router.get("/{instanceId}/instance-roles", response_model=PaginatedResponse) +@limiter.limit("30/minute") +async def getInstanceRoles( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + context: RequestContext = Depends(getRequestContext) +) -> PaginatedResponse: + """ + Get all roles for this feature instance. + Requires feature admin permission. + """ + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + + # Get instance-specific roles (mandateId set, featureInstanceId matches) + roles = rootInterface.db.getRecordset( + Role, + recordFilter={ + "featureCode": "trustee", + "featureInstanceId": instanceId + } + ) + + return PaginatedResponse( + items=roles, + pagination=None + ) + + +@router.get("/{instanceId}/instance-roles/{roleId}", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def getInstanceRole( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + roleId: str = Path(..., description="Role ID"), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """Get a specific instance role.""" + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + role = rootInterface.db.getRecord(Role, roleId) + + if not role: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found") + + # Verify role belongs to this instance + if role.get("featureInstanceId") != instanceId: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") + + return role + + +@router.get("/{instanceId}/instance-roles/{roleId}/rules", response_model=PaginatedResponse) +@limiter.limit("30/minute") +async def getInstanceRoleRules( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + roleId: str = Path(..., description="Role ID"), + context: RequestContext = Depends(getRequestContext) +) -> PaginatedResponse: + """ + Get all AccessRules for a specific instance role. + Requires feature admin permission. + """ + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + + # Verify role belongs to this instance + role = rootInterface.db.getRecord(Role, roleId) + if not role or role.get("featureInstanceId") != instanceId: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") + + # Get AccessRules for this role + rules = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": roleId} + ) + + return PaginatedResponse( + items=rules, + pagination=None + ) + + +@router.post("/{instanceId}/instance-roles/{roleId}/rules", response_model=Dict[str, Any], status_code=201) +@limiter.limit("10/minute") +async def createInstanceRoleRule( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + roleId: str = Path(..., description="Role ID"), + ruleData: Dict[str, Any] = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Create a new AccessRule for an instance role. + Requires feature admin permission. + """ + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + + # Verify role belongs to this instance + role = rootInterface.db.getRecord(Role, roleId) + if not role or role.get("featureInstanceId") != instanceId: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") + + # Create the rule + try: + contextStr = ruleData.get("context", "UI") + if isinstance(contextStr, str): + contextEnum = AccessRuleContext(contextStr.upper()) + else: + contextEnum = contextStr + + newRule = AccessRule( + roleId=roleId, + context=contextEnum, + item=ruleData.get("item"), + view=ruleData.get("view", False), + read=ruleData.get("read"), + create=ruleData.get("create"), + update=ruleData.get("update"), + delete=ruleData.get("delete"), + ) + + created = rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) + return created + + except Exception as e: + logger.error(f"Error creating AccessRule: {e}") + raise HTTPException(status_code=400, detail=f"Failed to create rule: {str(e)}") + + +@router.put("/{instanceId}/instance-roles/{roleId}/rules/{ruleId}", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def updateInstanceRoleRule( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + roleId: str = Path(..., description="Role ID"), + ruleId: str = Path(..., description="Rule ID"), + ruleData: Dict[str, Any] = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Update an AccessRule for an instance role. + Only view, read, create, update, delete can be changed. + Requires feature admin permission. + """ + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + + # Verify role belongs to this instance + role = rootInterface.db.getRecord(Role, roleId) + if not role or role.get("featureInstanceId") != instanceId: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") + + # Verify rule belongs to role + existingRule = rootInterface.db.getRecord(AccessRule, ruleId) + if not existingRule or existingRule.get("roleId") != roleId: + raise HTTPException(status_code=404, detail=f"Rule {ruleId} not found for this role") + + # Update only allowed fields + updateData = {} + if "view" in ruleData: + updateData["view"] = ruleData["view"] + if "read" in ruleData: + updateData["read"] = ruleData["read"] + if "create" in ruleData: + updateData["create"] = ruleData["create"] + if "update" in ruleData: + updateData["update"] = ruleData["update"] + if "delete" in ruleData: + updateData["delete"] = ruleData["delete"] + + if not updateData: + return existingRule + + try: + updated = rootInterface.db.recordUpdate(AccessRule, ruleId, updateData) + return updated + except Exception as e: + logger.error(f"Error updating AccessRule: {e}") + raise HTTPException(status_code=400, detail=f"Failed to update rule: {str(e)}") + + +@router.delete("/{instanceId}/instance-roles/{roleId}/rules/{ruleId}") +@limiter.limit("10/minute") +async def deleteInstanceRoleRule( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + roleId: str = Path(..., description="Role ID"), + ruleId: str = Path(..., description="Rule ID"), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Delete an AccessRule for an instance role. + Requires feature admin permission. + """ + mandateId = await _validateInstanceAdmin(instanceId, context) + + rootInterface = getRootInterface() + + # Verify role belongs to this instance + role = rootInterface.db.getRecord(Role, roleId) + if not role or role.get("featureInstanceId") != instanceId: + raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") + + # Verify rule belongs to role + existingRule = rootInterface.db.getRecord(AccessRule, ruleId) + if not existingRule or existingRule.get("roleId") != roleId: + raise HTTPException(status_code=404, detail=f"Rule {ruleId} not found for this role") + + try: + rootInterface.db.recordDelete(AccessRule, ruleId) + return {"message": f"Rule {ruleId} deleted"} + except Exception as e: + logger.error(f"Error deleting AccessRule: {e}") + raise HTTPException(status_code=400, detail=f"Failed to delete rule: {str(e)}") diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index f16a9bc7..dcd32bbc 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -311,6 +311,50 @@ async def getAccessRules( ) +@router.get("/rules/by-role/{roleId}", response_model=PaginatedResponse) +@limiter.limit("30/minute") +async def getAccessRulesByRole( + request: Request, + roleId: str = Path(..., description="Role ID to get rules for"), + currentUser: User = Depends(requireSysAdmin) +) -> PaginatedResponse: + """ + Get all access rules for a specific role. + MULTI-TENANT: SysAdmin-only. + + Path Parameters: + - roleId: The role ID to get rules for + + Returns: + - List of AccessRule objects for the specified role + """ + try: + interface = getRootInterface() + + # Build filter for roleId + recordFilter = {"roleId": roleId} + + # Get rules from database + rules = interface.db.getRecordset(AccessRule, recordFilter=recordFilter) + + # Convert to AccessRule objects + ruleObjects = [AccessRule(**rule) for rule in rules] + + return PaginatedResponse( + items=[rule.model_dump() for rule in ruleObjects], + pagination=None + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting access rules for role {roleId}: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to get access rules for role: {str(e)}" + ) + + @router.get("/rules/{ruleId}", response_model=dict) @limiter.limit("30/minute") async def getAccessRule( From 4de962d7d6bee3ba3d2b3a254c11755b3cfd2ac6 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 24 Jan 2026 09:58:15 +0100 Subject: [PATCH 21/32] access rules editor fixed --- .../features/trustee/routeFeatureTrustee.py | 30 ++++++++++--------- modules/routes/routeAdminFeatures.py | 28 +++++++++-------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 9e30951a..4e796a7e 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -1280,11 +1280,13 @@ async def getInstanceRole( mandateId = await _validateInstanceAdmin(instanceId, context) rootInterface = getRootInterface() - role = rootInterface.db.getRecord(Role, roleId) + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) - if not role: + if not roles: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") + role = roles[0] + # Verify role belongs to this instance if role.get("featureInstanceId") != instanceId: raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") @@ -1309,8 +1311,8 @@ async def getInstanceRoleRules( rootInterface = getRootInterface() # Verify role belongs to this instance - role = rootInterface.db.getRecord(Role, roleId) - if not role or role.get("featureInstanceId") != instanceId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roles or roles[0].get("featureInstanceId") != instanceId: raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") # Get AccessRules for this role @@ -1343,8 +1345,8 @@ async def createInstanceRoleRule( rootInterface = getRootInterface() # Verify role belongs to this instance - role = rootInterface.db.getRecord(Role, roleId) - if not role or role.get("featureInstanceId") != instanceId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roles or roles[0].get("featureInstanceId") != instanceId: raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") # Create the rule @@ -1394,13 +1396,13 @@ async def updateInstanceRoleRule( rootInterface = getRootInterface() # Verify role belongs to this instance - role = rootInterface.db.getRecord(Role, roleId) - if not role or role.get("featureInstanceId") != instanceId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roles or roles[0].get("featureInstanceId") != instanceId: raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") # Verify rule belongs to role - existingRule = rootInterface.db.getRecord(AccessRule, ruleId) - if not existingRule or existingRule.get("roleId") != roleId: + existingRules = rootInterface.db.getRecordset(AccessRule, recordFilter={"id": ruleId}) + if not existingRules or existingRules[0].get("roleId") != roleId: raise HTTPException(status_code=404, detail=f"Rule {ruleId} not found for this role") # Update only allowed fields @@ -1445,13 +1447,13 @@ async def deleteInstanceRoleRule( rootInterface = getRootInterface() # Verify role belongs to this instance - role = rootInterface.db.getRecord(Role, roleId) - if not role or role.get("featureInstanceId") != instanceId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if not roles or roles[0].get("featureInstanceId") != instanceId: raise HTTPException(status_code=404, detail=f"Role {roleId} not found in this instance") # Verify rule belongs to role - existingRule = rootInterface.db.getRecord(AccessRule, ruleId) - if not existingRule or existingRule.get("roleId") != roleId: + existingRules = rootInterface.db.getRecordset(AccessRule, recordFilter={"id": ruleId}) + if not existingRules or existingRules[0].get("roleId") != roleId: raise HTTPException(status_code=404, detail=f"Rule {ruleId} not found for this role") try: diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 9cb023c6..265a3a5c 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -164,8 +164,8 @@ async def getMyFeatureInstances( "_mandateId": mandateId # Temporary for grouping } - # Get user's role in this instance - userRole = _getUserRoleInInstance(rootInterface, str(context.user.id), str(instance.id)) + # Get user's roles in this instance (can have multiple) + userRoles = _getUserRolesInInstance(rootInterface, str(context.user.id), str(instance.id)) # Get permissions for this instance permissions = _getInstancePermissions(rootInterface, str(context.user.id), str(instance.id)) @@ -177,7 +177,7 @@ async def getMyFeatureInstances( "mandateId": mandateId, "mandateName": mandatesMap[mandateId]["name"], "instanceLabel": instance.label, - "userRole": userRole, + "userRoles": userRoles, "permissions": permissions }) @@ -196,8 +196,8 @@ async def getMyFeatureInstances( ) -def _getUserRoleInInstance(rootInterface, userId: str, instanceId: str) -> str: - """Get the user's primary role label in a feature instance.""" +def _getUserRolesInInstance(rootInterface, userId: str, instanceId: str) -> List[str]: + """Get all role labels for a user in a feature instance.""" try: from modules.datamodels.datamodelRbac import Role from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole @@ -218,15 +218,19 @@ def _getUserRoleInInstance(rootInterface, userId: str, instanceId: str) -> str: ) if featureAccessRoles: - roleId = featureAccessRoles[0].get("roleId") - roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) - if roles: - return roles[0].get("roleLabel", "user") + # Get ALL roles, not just the first one + roleLabels = [] + for far in featureAccessRoles: + roleId = far.get("roleId") + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles: + roleLabels.append(roles[0].get("roleLabel", "user")) + return roleLabels if roleLabels else ["user"] - return "user" # Default + return ["user"] # Default except Exception as e: - logger.debug(f"Error getting user role: {e}") - return "user" + logger.debug(f"Error getting user roles: {e}") + return ["user"] def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict[str, Any]: From df4c60fc990331d0291bd5b3ddaedcde7cac8bd3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 24 Jan 2026 18:01:28 +0100 Subject: [PATCH 22/32] fixes --- app.py | 10 + modules/datamodels/datamodelUam.py | 35 ++- .../chatbot/interfaceFeatureChatbot.py | 7 +- .../realEstate/interfaceFeatureRealEstate.py | 7 +- .../trustee/datamodelFeatureTrustee.py | 127 ++++------- .../trustee/interfaceFeatureTrustee.py | 171 +++++++------- .../features/trustee/routeFeatureTrustee.py | 208 +++++++++++++----- modules/interfaces/interfaceDbApp.py | 62 ++++-- modules/interfaces/interfaceDbChat.py | 7 +- modules/interfaces/interfaceFeatures.py | 39 +++- modules/interfaces/interfaceRbac.py | 42 +++- modules/routes/routeAdminFeatures.py | 178 +++++++++++++-- modules/routes/routeAdminRbacExport.py | 8 +- modules/routes/routeAdminRbacRoles.py | 24 +- modules/routes/routeAdminRbacRules.py | 28 +-- modules/routes/routeDataConnections.py | 4 +- modules/routes/routeDataMandates.py | 8 +- modules/routes/routeDataUsers.py | 4 +- modules/routes/routeGdpr.py | 8 +- modules/routes/routeInvitations.py | 12 +- modules/routes/routeMessaging.py | 28 +-- modules/routes/routeSecurityLocal.py | 30 +-- modules/routes/routeVoiceGoogle.py | 2 +- tests/functional/test03_ai_operations.py | 7 +- tests/functional/test04_ai_behavior.py | 5 +- .../test05_workflow_with_documents.py | 5 +- .../test06_workflow_prompt_variations.py | 5 +- .../test09_document_generation_formats.py | 5 +- .../test10_document_generation_formats.py | 5 +- .../test11_code_generation_formats.py | 5 +- 30 files changed, 705 insertions(+), 381 deletions(-) diff --git a/app.py b/app.py index 57274338..a73cbcbc 100644 --- a/app.py +++ b/app.py @@ -335,6 +335,15 @@ async def lifespan(app: FastAPI): logger.info("Application has been shut down") +# Custom function to generate readable operation IDs for Swagger UI +# Uses snake_case function names directly instead of auto-generated IDs +def _generateOperationId(route) -> str: + """Generate operation ID from route function name (snake_case).""" + if hasattr(route, "endpoint") and hasattr(route.endpoint, "__name__"): + return route.endpoint.__name__ + return route.name if route.name else "unknown" + + # START APP app = FastAPI( title="PowerOn | Data Platform API", @@ -343,6 +352,7 @@ app = FastAPI( swagger_ui_init_oauth={ "usePkceWithAuthorizationCodeGrant": True, }, + generate_unique_id_function=_generateOperationId, ) # Configure OpenAPI security scheme for Swagger UI diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index 42659159..b7878532 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -166,15 +166,38 @@ class User(BaseModel): json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False} ) language: str = Field( - default="en", - description="Preferred language of the user", + default="de", + description="Preferred language of the user (ISO 639-1 code: de, en, fr, it)", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True, "frontend_options": [ - {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, - {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, - {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, - {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, + {"value": "de", "label": {"en": "Deutsch", "de": "Deutsch", "fr": "Allemand"}}, + {"value": "en", "label": {"en": "English", "de": "Englisch", "fr": "Anglais"}}, + {"value": "fr", "label": {"en": "Français", "de": "Französisch", "fr": "Français"}}, + {"value": "it", "label": {"en": "Italiano", "de": "Italienisch", "fr": "Italien"}}, ]} ) + + @field_validator('language', mode='before') + @classmethod + def _normalizeLanguage(cls, v): + """Normalize language to valid ISO 639-1 code.""" + if v is None: + return "de" + # Map common variations to standard codes + langMap = { + 'english': 'en', 'englisch': 'en', + 'german': 'de', 'deutsch': 'de', + 'french': 'fr', 'französisch': 'fr', 'francais': 'fr', + 'italian': 'it', 'italienisch': 'it', 'italiano': 'it', + } + normalized = str(v).lower().strip() + if normalized in langMap: + return langMap[normalized] + # If already a valid code, return as-is + if normalized in ['de', 'en', 'fr', 'it']: + return normalized + # Default fallback + return "de" + enabled: bool = Field( default=True, description="Indicates whether the user is enabled", diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 685a1a4e..7db04c46 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -282,9 +282,10 @@ class ChatObjects: if not self.userId: raise ValueError("Invalid user context: id is required") - # mandateId can be None for sysadmins performing cross-mandate operations - if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): - raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. + # Users are NOT assigned to mandates by design - they get mandate context from the request. + # sysAdmin users can additionally perform cross-mandate operations. + # Without mandateId, operations will be filtered to accessible mandates via RBAC. # Add language settings self.userLanguage = currentUser.language # Default user language diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 545be2c0..5a8c6d70 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -126,9 +126,10 @@ class RealEstateObjects: if not self.userId: raise ValueError("Invalid user context: id is required") - # mandateId can be None for sysadmins performing cross-mandate operations - if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): - raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. + # Users are NOT assigned to mandates by design - they get mandate context from the request. + # sysAdmin users can additionally perform cross-mandate operations. + # Without mandateId, operations will be filtered to accessible mandates via RBAC. # Initialize RBAC interface if not self.currentUser: diff --git a/modules/features/trustee/datamodelFeatureTrustee.py b/modules/features/trustee/datamodelFeatureTrustee.py index c64ec506..8b13dff1 100644 --- a/modules/features/trustee/datamodelFeatureTrustee.py +++ b/modules/features/trustee/datamodelFeatureTrustee.py @@ -279,7 +279,12 @@ registerModelLabels( class TrusteeDocument(BaseModel): - """Contains document references and receipts for bookings.""" + """Contains document references and receipts for bookings. + + Note: organisationId and contractId removed as per architecture decision: + - The feature instance IS the organisation + - Contracts are eliminated from the model + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique document ID", @@ -289,25 +294,6 @@ class TrusteeDocument(BaseModel): "frontend_required": False } ) - organisationId: str = Field( - description="Reference to TrusteeOrganisation.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/organisations/options" - } - ) - contractId: str = Field( - description="Reference to TrusteeContract.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/contracts/options", - "frontend_depends_on": "organisationId" - } - ) documentData: Optional[bytes] = Field( default=None, description="The file content (binary)", @@ -332,30 +318,27 @@ class TrusteeDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": [ - {"value": "application/pdf", "label": {"en": "PDF", "fr": "PDF", "de": "PDF"}}, - {"value": "image/jpeg", "label": {"en": "JPEG", "fr": "JPEG", "de": "JPEG"}}, - {"value": "image/png", "label": {"en": "PNG", "fr": "PNG", "de": "PNG"}}, - {"value": "application/octet-stream", "label": {"en": "Other", "fr": "Autre", "de": "Andere"}}, - ] + "frontend_options": "/api/trustee/mime-types/options" } ) mandateId: Optional[str] = Field( default=None, - description="Mandate ID", + description="Mandate ID (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) featureInstanceId: Optional[str] = Field( default=None, - description="Feature Instance ID for instance-level isolation", + description="Feature Instance ID for instance-level isolation (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) # System attributes are automatically set by DatabaseConnector @@ -366,8 +349,6 @@ registerModelLabels( {"en": "Document", "fr": "Document", "de": "Dokument"}, { "id": {"en": "ID", "fr": "ID", "de": "ID"}, - "organisationId": {"en": "Organisation", "fr": "Organisation", "de": "Organisation"}, - "contractId": {"en": "Contract", "fr": "Contrat", "de": "Vertrag"}, "documentData": {"en": "Document Data", "fr": "Données du document", "de": "Dokumentdaten"}, "documentName": {"en": "Document Name", "fr": "Nom du document", "de": "Dokumentname"}, "documentMimeType": {"en": "MIME Type", "fr": "Type MIME", "de": "MIME-Typ"}, @@ -378,7 +359,12 @@ registerModelLabels( class TrusteePosition(BaseModel): - """Contains booking positions (expense entries).""" + """Contains booking positions (expense entries). + + Note: organisationId and contractId removed as per architecture decision: + - The feature instance IS the organisation + - Contracts are eliminated from the model + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique position ID", @@ -388,25 +374,6 @@ class TrusteePosition(BaseModel): "frontend_required": False } ) - organisationId: str = Field( - description="Reference to TrusteeOrganisation.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/organisations/options" - } - ) - contractId: str = Field( - description="Reference to TrusteeContract.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/contracts/options", - "frontend_depends_on": "organisationId" - } - ) valuta: Optional[str] = Field( default=None, description="Value date (ISO format: YYYY-MM-DD)", @@ -520,20 +487,22 @@ class TrusteePosition(BaseModel): ) mandateId: Optional[str] = Field( default=None, - description="Mandate ID", + description="Mandate ID (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) featureInstanceId: Optional[str] = Field( default=None, - description="Feature Instance ID for instance-level isolation", + description="Feature Instance ID for instance-level isolation (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) # System attributes are automatically set by DatabaseConnector @@ -544,8 +513,6 @@ registerModelLabels( {"en": "Position", "fr": "Position", "de": "Position"}, { "id": {"en": "ID", "fr": "ID", "de": "ID"}, - "organisationId": {"en": "Organisation", "fr": "Organisation", "de": "Organisation"}, - "contractId": {"en": "Contract", "fr": "Contrat", "de": "Vertrag"}, "valuta": {"en": "Value Date", "fr": "Date de valeur", "de": "Valutadatum"}, "transactionDateTime": {"en": "Transaction Date/Time", "fr": "Date/Heure de transaction", "de": "Transaktionszeitpunkt"}, "company": {"en": "Company", "fr": "Entreprise", "de": "Firma"}, @@ -564,7 +531,12 @@ registerModelLabels( class TrusteePositionDocument(BaseModel): - """Cross-reference table linking positions to documents (many-to-many).""" + """Cross-reference table linking positions to documents (many-to-many). + + Note: organisationId and contractId removed as per architecture decision: + - The feature instance IS the organisation + - Contracts are eliminated from the model + """ id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique link ID", @@ -574,33 +546,13 @@ class TrusteePositionDocument(BaseModel): "frontend_required": False } ) - organisationId: str = Field( - description="Reference to TrusteeOrganisation.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/organisations/options" - } - ) - contractId: str = Field( - description="Reference to TrusteeContract.id", - json_schema_extra={ - "frontend_type": "select", - "frontend_readonly": False, - "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/contracts/options", - "frontend_depends_on": "organisationId" - } - ) documentId: str = Field( description="Reference to TrusteeDocument.id", json_schema_extra={ "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/documents/options", - "frontend_depends_on": "contractId" + "frontend_options": "/api/trustee/{instanceId}/documents/options" } ) positionId: str = Field( @@ -609,26 +561,27 @@ class TrusteePositionDocument(BaseModel): "frontend_type": "select", "frontend_readonly": False, "frontend_required": True, - "frontend_options": "/api/trustee/{instanceId}/positions/options", - "frontend_depends_on": "contractId" + "frontend_options": "/api/trustee/{instanceId}/positions/options" } ) mandateId: Optional[str] = Field( default=None, - description="Mandate ID", + description="Mandate ID (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) featureInstanceId: Optional[str] = Field( default=None, - description="Feature Instance ID for instance-level isolation", + description="Feature Instance ID for instance-level isolation (auto-set from context)", json_schema_extra={ "frontend_type": "text", "frontend_readonly": True, - "frontend_required": False + "frontend_required": False, + "frontend_hidden": True } ) # System attributes are automatically set by DatabaseConnector @@ -639,8 +592,6 @@ registerModelLabels( {"en": "Position-Document Link", "fr": "Lien Position-Document", "de": "Position-Dokument-Verknüpfung"}, { "id": {"en": "ID", "fr": "ID", "de": "ID"}, - "organisationId": {"en": "Organisation", "fr": "Organisation", "de": "Organisation"}, - "contractId": {"en": "Contract", "fr": "Contrat", "de": "Vertrag"}, "documentId": {"en": "Document", "fr": "Document", "de": "Dokument"}, "positionId": {"en": "Position", "fr": "Position", "de": "Position"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index dbd8c8aa..7fc90bdb 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -110,9 +110,10 @@ class TrusteeObjects: if not self.userId: raise ValueError("Invalid user context: id is required") - # mandateId can be None for sysadmins performing cross-mandate operations - if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): - raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. + # Users are NOT assigned to mandates by design - they get mandate context from the request. + # sysAdmin users can additionally perform cross-mandate operations. + # Without mandateId, operations will be filtered to accessible mandates via RBAC. self.userLanguage = currentUser.language @@ -734,18 +735,19 @@ class TrusteeObjects: # ===== Document CRUD ===== def createDocument(self, data: Dict[str, Any]) -> Optional[TrusteeDocument]: - """Create a new document.""" - organisationId = data.get("organisationId") - contractId = data.get("contractId") + """Create a new document. - # Check combined permission (system RBAC + feature-level) - if not self.checkCombinedPermission(TrusteeDocument, "create", organisationId, contractId): - logger.warning(f"User {self.userId} lacks permission to create document in org {organisationId}") + Note: organisationId and contractId removed - feature instance IS the organisation. + Permission is checked via system RBAC (feature-level access). + """ + # Check system RBAC permission + if not self.checkCombinedPermission(TrusteeDocument, "create"): + logger.warning(f"User {self.userId} lacks permission to create document") return None + # Auto-set context fields data["mandateId"] = self.mandateId - if "featureInstanceId" not in data: - data["featureInstanceId"] = self.featureInstanceId + data["featureInstanceId"] = self.featureInstanceId import uuid documentId = data.get("id") or str(uuid.uuid4()) @@ -790,20 +792,22 @@ class TrusteeObjects: # This applies userreport filtering (only own records) records = self.filterRecordsByTrusteeAccess(records, TrusteeDocument) - # Remove binary data from responses + # Convert dicts to Pydantic objects (remove binary data and internal fields) + pydanticItems = [] for record in records: - record.pop("documentData", None) + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} + pydanticItems.append(TrusteeDocument(**cleanedRecord)) - totalItems = len(records) + totalItems = len(pydanticItems) if params: pageSize = params.pageSize or 20 page = params.page or 1 startIdx = (page - 1) * pageSize endIdx = startIdx + pageSize - items = records[startIdx:endIdx] + items = pydanticItems[startIdx:endIdx] totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 else: - items = records + items = pydanticItems totalPages = 1 page = 1 pageSize = totalItems @@ -835,8 +839,11 @@ class TrusteeObjects: return result def updateDocument(self, documentId: str, data: Dict[str, Any]) -> Optional[TrusteeDocument]: - """Update a document.""" - # Get existing document to check organisation and creator + """Update a document. + + Note: organisationId and contractId removed - feature instance IS the organisation. + """ + # Get existing document to check creator existingRecords = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) existing = existingRecords[0] if existingRecords else None @@ -844,14 +851,11 @@ class TrusteeObjects: logger.warning(f"Document {documentId} not found") return None - organisationId = existing.get("organisationId") - contractId = existing.get("contractId") createdBy = existing.get("_createdBy") - # Check combined permission (system RBAC + feature-level) - # For userreport, this checks if they created the record - if not self.checkCombinedPermission(TrusteeDocument, "update", organisationId, contractId, createdBy): - logger.warning(f"User {self.userId} lacks permission to update document in org {organisationId}") + # Check system RBAC permission (userreport can only edit their own records) + if not self.checkCombinedPermission(TrusteeDocument, "update", recordCreatedBy=createdBy): + logger.warning(f"User {self.userId} lacks permission to update document") return None data["id"] = documentId @@ -862,8 +866,11 @@ class TrusteeObjects: return TrusteeDocument(**cleanedRecord) def deleteDocument(self, documentId: str) -> bool: - """Delete a document.""" - # Get existing document to check organisation and creator + """Delete a document. + + Note: organisationId and contractId removed - feature instance IS the organisation. + """ + # Get existing document to check creator existingRecords = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) existing = existingRecords[0] if existingRecords else None @@ -871,14 +878,11 @@ class TrusteeObjects: logger.warning(f"Document {documentId} not found") return False - organisationId = existing.get("organisationId") - contractId = existing.get("contractId") createdBy = existing.get("_createdBy") - # Check combined permission (system RBAC + feature-level) - # For userreport, this checks if they created the record - if not self.checkCombinedPermission(TrusteeDocument, "delete", organisationId, contractId, createdBy): - logger.warning(f"User {self.userId} lacks permission to delete document in org {organisationId}") + # Check system RBAC permission (userreport can only delete their own records) + if not self.checkCombinedPermission(TrusteeDocument, "delete", recordCreatedBy=createdBy): + logger.warning(f"User {self.userId} lacks permission to delete document") return False return self.db.recordDelete(TrusteeDocument, documentId) @@ -886,18 +890,19 @@ class TrusteeObjects: # ===== Position CRUD ===== def createPosition(self, data: Dict[str, Any]) -> Optional[TrusteePosition]: - """Create a new position.""" - organisationId = data.get("organisationId") - contractId = data.get("contractId") + """Create a new position. - # Check combined permission (system RBAC + feature-level) - if not self.checkCombinedPermission(TrusteePosition, "create", organisationId, contractId): - logger.warning(f"User {self.userId} lacks permission to create position in org {organisationId}") + Note: organisationId and contractId removed - feature instance IS the organisation. + Permission is checked via system RBAC (feature-level access). + """ + # Check system RBAC permission + if not self.checkCombinedPermission(TrusteePosition, "create"): + logger.warning(f"User {self.userId} lacks permission to create position") return None + # Auto-set context fields data["mandateId"] = self.mandateId - if "featureInstanceId" not in data: - data["featureInstanceId"] = self.featureInstanceId + data["featureInstanceId"] = self.featureInstanceId # Calculate VAT amount if not provided if "vatAmount" not in data or data.get("vatAmount") == 0: @@ -936,16 +941,22 @@ class TrusteeObjects: # This applies userreport filtering (only own records) records = self.filterRecordsByTrusteeAccess(records, TrusteePosition) - totalItems = len(records) + # Convert dicts to Pydantic objects (remove internal fields) + pydanticItems = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + pydanticItems.append(TrusteePosition(**cleanedRecord)) + + totalItems = len(pydanticItems) if params: pageSize = params.pageSize or 20 page = params.page or 1 startIdx = (page - 1) * pageSize endIdx = startIdx + pageSize - items = records[startIdx:endIdx] + items = pydanticItems[startIdx:endIdx] totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 else: - items = records + items = pydanticItems totalPages = 1 page = 1 pageSize = totalItems @@ -987,8 +998,11 @@ class TrusteeObjects: return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] def updatePosition(self, positionId: str, data: Dict[str, Any]) -> Optional[TrusteePosition]: - """Update a position.""" - # Get existing position to check organisation and creator + """Update a position. + + Note: organisationId and contractId removed - feature instance IS the organisation. + """ + # Get existing position to check creator existingRecords = self.db.getRecordset(TrusteePosition, recordFilter={"id": positionId}) existing = existingRecords[0] if existingRecords else None @@ -996,14 +1010,11 @@ class TrusteeObjects: logger.warning(f"Position {positionId} not found") return None - organisationId = existing.get("organisationId") - contractId = existing.get("contractId") createdBy = existing.get("_createdBy") - # Check combined permission (system RBAC + feature-level) - # For userreport, this checks if they created the record - if not self.checkCombinedPermission(TrusteePosition, "update", organisationId, contractId, createdBy): - logger.warning(f"User {self.userId} lacks permission to update position in org {organisationId}") + # Check system RBAC permission (userreport can only edit their own records) + if not self.checkCombinedPermission(TrusteePosition, "update", recordCreatedBy=createdBy): + logger.warning(f"User {self.userId} lacks permission to update position") return None data["id"] = positionId @@ -1013,8 +1024,11 @@ class TrusteeObjects: return TrusteePosition(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) def deletePosition(self, positionId: str) -> bool: - """Delete a position.""" - # Get existing position to check organisation and creator + """Delete a position. + + Note: organisationId and contractId removed - feature instance IS the organisation. + """ + # Get existing position to check creator existingRecords = self.db.getRecordset(TrusteePosition, recordFilter={"id": positionId}) existing = existingRecords[0] if existingRecords else None @@ -1022,14 +1036,11 @@ class TrusteeObjects: logger.warning(f"Position {positionId} not found") return False - organisationId = existing.get("organisationId") - contractId = existing.get("contractId") createdBy = existing.get("_createdBy") - # Check combined permission (system RBAC + feature-level) - # For userreport, this checks if they created the record - if not self.checkCombinedPermission(TrusteePosition, "delete", organisationId, contractId, createdBy): - logger.warning(f"User {self.userId} lacks permission to delete position in org {organisationId}") + # Check system RBAC permission (userreport can only delete their own records) + if not self.checkCombinedPermission(TrusteePosition, "delete", recordCreatedBy=createdBy): + logger.warning(f"User {self.userId} lacks permission to delete position") return False return self.db.recordDelete(TrusteePosition, positionId) @@ -1037,18 +1048,19 @@ class TrusteeObjects: # ===== Position-Document Link CRUD ===== def createPositionDocument(self, data: Dict[str, Any]) -> Optional[TrusteePositionDocument]: - """Create a new position-document link.""" - organisationId = data.get("organisationId") - contractId = data.get("contractId") + """Create a new position-document link. - # Check combined permission (system RBAC + feature-level) - if not self.checkCombinedPermission(TrusteePositionDocument, "create", organisationId, contractId): - logger.warning(f"User {self.userId} lacks permission to create position-document link in org {organisationId}") + Note: organisationId and contractId removed - feature instance IS the organisation. + Permission is checked via system RBAC (feature-level access). + """ + # Check system RBAC permission + if not self.checkCombinedPermission(TrusteePositionDocument, "create"): + logger.warning(f"User {self.userId} lacks permission to create position-document link") return None + # Auto-set context fields data["mandateId"] = self.mandateId - if "featureInstanceId" not in data: - data["featureInstanceId"] = self.featureInstanceId + data["featureInstanceId"] = self.featureInstanceId import uuid linkId = data.get("id") or str(uuid.uuid4()) @@ -1132,8 +1144,11 @@ class TrusteeObjects: return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] def deletePositionDocument(self, linkId: str) -> bool: - """Delete a position-document link.""" - # Get existing link to check organisation and creator + """Delete a position-document link. + + Note: organisationId and contractId removed - feature instance IS the organisation. + """ + # Get existing link to check creator existingRecords = self.db.getRecordset(TrusteePositionDocument, recordFilter={"id": linkId}) existing = existingRecords[0] if existingRecords else None @@ -1141,14 +1156,11 @@ class TrusteeObjects: logger.warning(f"Position-document link {linkId} not found") return False - organisationId = existing.get("organisationId") - contractId = existing.get("contractId") createdBy = existing.get("_createdBy") - # Check combined permission (system RBAC + feature-level) - # For userreport, this checks if they created the record - if not self.checkCombinedPermission(TrusteePositionDocument, "delete", organisationId, contractId, createdBy): - logger.warning(f"User {self.userId} lacks permission to delete position-document link in org {organisationId}") + # Check system RBAC permission (userreport can only delete their own records) + if not self.checkCombinedPermission(TrusteePositionDocument, "delete", recordCreatedBy=createdBy): + logger.warning(f"User {self.userId} lacks permission to delete position-document link") return False return self.db.recordDelete(TrusteePositionDocument, linkId) @@ -1317,6 +1329,15 @@ class TrusteeObjects: if accessLevel == AccessLevel.ALL: return records + # NEW: Feature-instance based access (new system) + # If featureInstanceId is set, user has access via FeatureAccess system. + # Data is already filtered by featureInstanceId in getRecordsetWithRBAC. + # The old TrusteeAccess system (organisation-based) is not used for + # feature-instance scoped data. + if self.featureInstanceId: + return records # User already has access to this feature instance + + # LEGACY: TrusteeAccess based filtering (for backwards compatibility) # Get all user's access records userAccess = self.getAllUserAccess(self.userId) diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 4e796a7e..f2b2ead2 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -9,13 +9,14 @@ URL Structure: /api/trustee/{instanceId}/{entity} - This ensures proper multi-tenant isolation at the URL level """ -from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Query, Response +from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Query, Response, UploadFile, File, Form from fastapi.responses import StreamingResponse from typing import List, Dict, Any, Optional from fastapi import status import logging import json import io +import base64 from modules.auth import limiter, getRequestContext, RequestContext from .interfaceFeatureTrustee import getInterface @@ -133,7 +134,7 @@ _TRUSTEE_ENTITY_MODELS = { @router.get("/{instanceId}/attributes/{entityType}") @limiter.limit("30/minute") -async def getEntityAttributes( +async def get_entity_attributes( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), entityType: str = Path(..., description="Entity type (e.g., TrusteeDocument)"), @@ -179,9 +180,44 @@ async def getEntityAttributes( # OPTIONS ENDPOINTS (for dropdowns) # ============================================================================ +@router.get("/mime-types/options", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def get_mime_type_options( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """Get supported MIME types from the document extraction service. + Returns: [{ value: "mime/type", label: "Description" }] + """ + from modules.services.serviceExtraction.subRegistry import ExtractorRegistry + + registry = ExtractorRegistry() + formats = registry.getSupportedFormats() + + # Collect all unique MIME types + allMimeTypes = set() + for mimeList in formats.get("mime_types", {}).values(): + allMimeTypes.update(mimeList) + + # Sort and create options with labels + result = [] + for mimeType in sorted(allMimeTypes): + # Create readable label from mime type + parts = mimeType.split("/") + if len(parts) == 2: + mainType, subType = parts + # Clean up subtype for label + label = subType.replace("vnd.", "").replace("x-", "").replace("-", " ").title() + result.append({"value": mimeType, "label": f"{label} ({mimeType})"}) + else: + result.append({"value": mimeType, "label": mimeType}) + + return result + + @router.get("/{instanceId}/organisations/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getOrganisationOptions( +async def get_organisation_options( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) @@ -196,7 +232,7 @@ async def getOrganisationOptions( @router.get("/{instanceId}/roles/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getRoleOptions( +async def get_role_options( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) @@ -211,7 +247,7 @@ async def getRoleOptions( @router.get("/{instanceId}/contracts/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getContractOptions( +async def get_contract_options( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), organisationId: Optional[str] = Query(None, description="Optional: Filter by organisation ID"), @@ -241,7 +277,7 @@ async def getContractOptions( @router.get("/{instanceId}/documents/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getDocumentOptions( +async def get_document_options( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) @@ -256,7 +292,7 @@ async def getDocumentOptions( @router.get("/{instanceId}/positions/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getPositionOptions( +async def get_position_options( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) @@ -266,8 +302,8 @@ async def getPositionOptions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(None) items = result.items if hasattr(result, 'items') else result - # Erstelle Label aus Datum, Firma und Beschreibung - def _makePositionLabel(p): + + def _makePositionLabel(p: TrusteePosition) -> str: parts = [] if p.valuta: parts.append(str(p.valuta)[:10]) # Datum ohne Zeit @@ -276,6 +312,7 @@ async def getPositionOptions( if p.desc: parts.append(p.desc[:30]) return " - ".join(parts) if parts else p.id + return [{"value": p.id, "label": _makePositionLabel(p)} for p in items] @@ -287,7 +324,7 @@ async def getPositionOptions( @router.get("/{instanceId}/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) @limiter.limit("30/minute") -async def getOrganisations( +async def get_organisations( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), @@ -317,7 +354,7 @@ async def getOrganisations( @router.get("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("30/minute") -async def getOrganisation( +async def get_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), @@ -335,7 +372,7 @@ async def getOrganisation( @router.post("/{instanceId}/organisations", response_model=TrusteeOrganisation, status_code=201) @limiter.limit("10/minute") -async def createOrganisation( +async def create_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeOrganisation = Body(...), @@ -353,7 +390,7 @@ async def createOrganisation( @router.put("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("10/minute") -async def updateOrganisation( +async def update_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), @@ -376,7 +413,7 @@ async def updateOrganisation( @router.delete("/{instanceId}/organisations/{orgId}") @limiter.limit("10/minute") -async def deleteOrganisation( +async def delete_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), @@ -400,7 +437,7 @@ async def deleteOrganisation( @router.get("/{instanceId}/roles", response_model=PaginatedResponse[TrusteeRole]) @limiter.limit("30/minute") -async def getRoles( +async def get_roles( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -430,7 +467,7 @@ async def getRoles( @router.get("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("30/minute") -async def getRole( +async def get_role( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), @@ -448,7 +485,7 @@ async def getRole( @router.post("/{instanceId}/roles", response_model=TrusteeRole, status_code=201) @limiter.limit("10/minute") -async def createRole( +async def create_role( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeRole = Body(...), @@ -466,7 +503,7 @@ async def createRole( @router.put("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("10/minute") -async def updateRole( +async def update_role( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), @@ -489,7 +526,7 @@ async def updateRole( @router.delete("/{instanceId}/roles/{roleId}") @limiter.limit("10/minute") -async def deleteRole( +async def delete_role( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), @@ -513,7 +550,7 @@ async def deleteRole( @router.get("/{instanceId}/access", response_model=PaginatedResponse[TrusteeAccess]) @limiter.limit("30/minute") -async def getAllAccess( +async def get_all_access( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -543,7 +580,7 @@ async def getAllAccess( @router.get("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("30/minute") -async def getAccess( +async def get_access( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), @@ -561,7 +598,7 @@ async def getAccess( @router.get("/{instanceId}/access/organisation/{orgId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") -async def getAccessByOrganisation( +async def get_access_by_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), @@ -576,7 +613,7 @@ async def getAccessByOrganisation( @router.get("/{instanceId}/access/user/{userId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") -async def getAccessByUser( +async def get_access_by_user( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), userId: str = Path(...), @@ -591,7 +628,7 @@ async def getAccessByUser( @router.post("/{instanceId}/access", response_model=TrusteeAccess, status_code=201) @limiter.limit("10/minute") -async def createAccess( +async def create_access( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeAccess = Body(...), @@ -609,7 +646,7 @@ async def createAccess( @router.put("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("10/minute") -async def updateAccess( +async def update_access( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), @@ -632,7 +669,7 @@ async def updateAccess( @router.delete("/{instanceId}/access/{accessId}") @limiter.limit("10/minute") -async def deleteAccess( +async def delete_access( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), @@ -656,7 +693,7 @@ async def deleteAccess( @router.get("/{instanceId}/contracts", response_model=PaginatedResponse[TrusteeContract]) @limiter.limit("30/minute") -async def getContracts( +async def get_contracts( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -686,7 +723,7 @@ async def getContracts( @router.get("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("30/minute") -async def getContract( +async def get_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), @@ -704,7 +741,7 @@ async def getContract( @router.get("/{instanceId}/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) @limiter.limit("30/minute") -async def getContractsByOrganisation( +async def get_contracts_by_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), @@ -719,7 +756,7 @@ async def getContractsByOrganisation( @router.post("/{instanceId}/contracts", response_model=TrusteeContract, status_code=201) @limiter.limit("10/minute") -async def createContract( +async def create_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeContract = Body(...), @@ -737,7 +774,7 @@ async def createContract( @router.put("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("10/minute") -async def updateContract( +async def update_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), @@ -760,7 +797,7 @@ async def updateContract( @router.delete("/{instanceId}/contracts/{contractId}") @limiter.limit("10/minute") -async def deleteContract( +async def delete_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), @@ -784,7 +821,7 @@ async def deleteContract( @router.get("/{instanceId}/documents", response_model=PaginatedResponse[TrusteeDocument]) @limiter.limit("30/minute") -async def getDocuments( +async def get_documents( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -814,7 +851,7 @@ async def getDocuments( @router.get("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("30/minute") -async def getDocument( +async def get_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), @@ -832,7 +869,7 @@ async def getDocument( @router.get("/{instanceId}/documents/{documentId}/data") @limiter.limit("10/minute") -async def getDocumentData( +async def get_document_data( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), @@ -859,7 +896,7 @@ async def getDocumentData( @router.get("/{instanceId}/documents/contract/{contractId}", response_model=List[TrusteeDocument]) @limiter.limit("30/minute") -async def getDocumentsByContract( +async def get_documents_by_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), @@ -874,17 +911,66 @@ async def getDocumentsByContract( @router.post("/{instanceId}/documents", response_model=TrusteeDocument, status_code=201) @limiter.limit("10/minute") -async def createDocument( +async def create_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), - data: TrusteeDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: - """Create a new document.""" + """Create a new document. Accepts JSON body with optional base64-encoded documentData.""" mandateId = await _validateInstanceAccess(instanceId, context) + # Parse JSON body + body = await request.json() + + # Handle documentData: convert base64 string to bytes if present + if "documentData" in body and body["documentData"]: + dataValue = body["documentData"] + if isinstance(dataValue, str): + # Base64-encoded data from frontend + try: + body["documentData"] = base64.b64decode(dataValue) + except Exception as e: + logger.warning(f"Failed to decode base64 documentData: {e}") + body["documentData"] = None + elif isinstance(dataValue, bytes): + # Already bytes + pass + else: + # Unknown format (e.g., File object serialized wrong) + body["documentData"] = None + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - result = interface.createDocument(data.model_dump()) + result = interface.createDocument(body) + if not result: + raise HTTPException(status_code=400, detail="Failed to create document") + return result + + +@router.post("/{instanceId}/documents/upload", response_model=TrusteeDocument, status_code=201) +@limiter.limit("10/minute") +async def upload_document( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + file: UploadFile = File(..., description="Document file"), + documentName: str = Form(..., description="Document name"), + documentMimeType: str = Form(default="application/octet-stream", description="MIME type"), + context: RequestContext = Depends(getRequestContext) +) -> TrusteeDocument: + """Upload a document with multipart/form-data.""" + mandateId = await _validateInstanceAccess(instanceId, context) + + # Read file content + fileContent = await file.read() + + # Build document data + docData = { + "documentName": documentName, + "documentMimeType": documentMimeType or file.content_type or "application/octet-stream", + "documentData": fileContent + } + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.createDocument(docData) if not result: raise HTTPException(status_code=400, detail="Failed to create document") return result @@ -892,7 +978,7 @@ async def createDocument( @router.put("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("10/minute") -async def updateDocument( +async def update_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), @@ -915,7 +1001,7 @@ async def updateDocument( @router.delete("/{instanceId}/documents/{documentId}") @limiter.limit("10/minute") -async def deleteDocument( +async def delete_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), @@ -939,7 +1025,7 @@ async def deleteDocument( @router.get("/{instanceId}/positions", response_model=PaginatedResponse[TrusteePosition]) @limiter.limit("30/minute") -async def getPositions( +async def get_positions( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -969,7 +1055,7 @@ async def getPositions( @router.get("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("30/minute") -async def getPosition( +async def get_position( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), @@ -987,7 +1073,7 @@ async def getPosition( @router.get("/{instanceId}/positions/contract/{contractId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") -async def getPositionsByContract( +async def get_positions_by_contract( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), @@ -1002,7 +1088,7 @@ async def getPositionsByContract( @router.get("/{instanceId}/positions/organisation/{orgId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") -async def getPositionsByOrganisation( +async def get_positions_by_organisation( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), @@ -1017,7 +1103,7 @@ async def getPositionsByOrganisation( @router.post("/{instanceId}/positions", response_model=TrusteePosition, status_code=201) @limiter.limit("10/minute") -async def createPosition( +async def create_position( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePosition = Body(...), @@ -1035,7 +1121,7 @@ async def createPosition( @router.put("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("10/minute") -async def updatePosition( +async def update_position( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), @@ -1058,7 +1144,7 @@ async def updatePosition( @router.delete("/{instanceId}/positions/{positionId}") @limiter.limit("10/minute") -async def deletePosition( +async def delete_position( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), @@ -1082,7 +1168,7 @@ async def deletePosition( @router.get("/{instanceId}/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) @limiter.limit("30/minute") -async def getPositionDocuments( +async def get_position_documents( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), @@ -1112,7 +1198,7 @@ async def getPositionDocuments( @router.get("/{instanceId}/position-documents/{linkId}", response_model=TrusteePositionDocument) @limiter.limit("30/minute") -async def getPositionDocument( +async def get_position_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), @@ -1130,7 +1216,7 @@ async def getPositionDocument( @router.get("/{instanceId}/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") -async def getDocumentsForPosition( +async def get_documents_for_position( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), @@ -1145,7 +1231,7 @@ async def getDocumentsForPosition( @router.get("/{instanceId}/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") -async def getPositionsForDocument( +async def get_positions_for_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), @@ -1160,7 +1246,7 @@ async def getPositionsForDocument( @router.post("/{instanceId}/position-documents", response_model=TrusteePositionDocument, status_code=201) @limiter.limit("10/minute") -async def createPositionDocument( +async def create_position_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePositionDocument = Body(...), @@ -1178,7 +1264,7 @@ async def createPositionDocument( @router.delete("/{instanceId}/position-documents/{linkId}") @limiter.limit("10/minute") -async def deletePositionDocument( +async def delete_position_document( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), @@ -1240,7 +1326,7 @@ async def _validateInstanceAdmin(instanceId: str, context: RequestContext) -> st @router.get("/{instanceId}/instance-roles", response_model=PaginatedResponse) @limiter.limit("30/minute") -async def getInstanceRoles( +async def get_instance_roles( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) @@ -1270,7 +1356,7 @@ async def getInstanceRoles( @router.get("/{instanceId}/instance-roles/{roleId}", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def getInstanceRole( +async def get_instance_role( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), @@ -1296,7 +1382,7 @@ async def getInstanceRole( @router.get("/{instanceId}/instance-roles/{roleId}/rules", response_model=PaginatedResponse) @limiter.limit("30/minute") -async def getInstanceRoleRules( +async def get_instance_role_rules( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), @@ -1329,7 +1415,7 @@ async def getInstanceRoleRules( @router.post("/{instanceId}/instance-roles/{roleId}/rules", response_model=Dict[str, Any], status_code=201) @limiter.limit("10/minute") -async def createInstanceRoleRule( +async def create_instance_role_rule( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), @@ -1378,7 +1464,7 @@ async def createInstanceRoleRule( @router.put("/{instanceId}/instance-roles/{roleId}/rules/{ruleId}", response_model=Dict[str, Any]) @limiter.limit("10/minute") -async def updateInstanceRoleRule( +async def update_instance_role_rule( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), @@ -1431,7 +1517,7 @@ async def updateInstanceRoleRule( @router.delete("/{instanceId}/instance-roles/{roleId}/rules/{ruleId}") @limiter.limit("10/minute") -async def deleteInstanceRoleRule( +async def delete_instance_role_rule( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 14a251c7..9c769e7c 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -107,11 +107,9 @@ class AppObjects: if not self.userId: raise ValueError("Invalid user context: id is required") - # mandateId is optional for isSysAdmin users doing system-level operations - isSysAdmin = getattr(currentUser, 'isSysAdmin', False) - if not self.mandateId and not isSysAdmin: - # Non-sysadmin users MUST have a mandateId for tenant-scoped operations - logger.warning(f"User {self.userId} has no mandateId context") + # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. + # Users are NOT assigned to mandates by design - they get mandate context from the request. + # sysAdmin users can additionally perform cross-mandate operations. # Add language settings self.userLanguage = currentUser.language # Default user language @@ -599,26 +597,51 @@ class AppObjects: logger.error(f"Error getting user by ID: {str(e)}") return None + def _getUserForAuthentication(self, username: str) -> Optional[Dict[str, Any]]: + """ + Get user record by username for authentication purposes. + + SECURITY NOTE: This method bypasses RBAC intentionally because: + 1. Users are NOT mandate-bound (Multi-Tenant Design) + 2. Authentication must work regardless of mandate context + 3. RBAC filtering for User table requires mandate context which doesn't exist at login time + + This method should ONLY be used for authentication flows. + For all other user queries, use getUserByUsername() which applies RBAC. + + Returns: + Full UserInDB record as dict, or None if not found + """ + try: + users = self.db.getRecordset(UserInDB, recordFilter={"username": username}) + if not users: + return None + return users[0] + except Exception as e: + logger.error(f"Error getting user for authentication: {str(e)}") + return None + def authenticateLocalUser(self, username: str, password: str) -> Optional[User]: - """Authenticates a user by username and password using local authentication.""" - # Clear the users table from cache and reload it + """ + Authenticates a user by username and password using local authentication. + + SECURITY NOTE: Uses _getUserForAuthentication() which bypasses RBAC. + This is intentional because users are mandate-independent. + """ + # Get full user record directly (bypasses RBAC - see _getUserForAuthentication docstring) + userRecord = self._getUserForAuthentication(username) - # Get user by username - user = self.getUserByUsername(username) - - if not user: + if not userRecord: raise ValueError("User not found") # Check if the user is enabled - if not user.enabled: + if not userRecord.get("enabled", True): raise ValueError("User is disabled") # Verify that the user has local authentication enabled - if user.authenticationAuthority != AuthAuthority.LOCAL: + authAuthority = userRecord.get("authenticationAuthority", AuthAuthority.LOCAL) + if authAuthority != AuthAuthority.LOCAL and authAuthority != AuthAuthority.LOCAL.value: raise ValueError("User does not have local authentication enabled") - - # Get the full user record with password hash for verification - userRecord = self.db.getRecordset(UserInDB, recordFilter={"id": user.id})[0] # Check if user has a reset token set (password reset required) if userRecord.get("resetToken"): @@ -630,7 +653,12 @@ class AppObjects: if not self._verifyPassword(password, userRecord["hashedPassword"]): raise ValueError("Invalid password") - return user + # Return clean User object (without password hash and internal fields) + cleanedUser = {k: v for k, v in userRecord.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} + # Ensure roleLabels is always a list + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + return User(**cleanedUser) def createUser( self, diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index de16d6af..25b3d3d6 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -282,9 +282,10 @@ class ChatObjects: if not self.userId: raise ValueError("Invalid user context: id is required") - # mandateId can be None for sysadmins performing cross-mandate operations - if not self.mandateId and not getattr(currentUser, 'isSysAdmin', False): - raise ValueError("Invalid user context: mandateId is required for non-sysadmin users") + # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. + # Users are NOT assigned to mandates by design - they get mandate context from the request. + # sysAdmin users can additionally perform cross-mandate operations. + # Without mandateId, operations will be filtered to accessible mandates via RBAC. # Add language settings self.userLanguage = currentUser.language # Default user language diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py index 697658b3..3a82a8f3 100644 --- a/modules/interfaces/interfaceFeatures.py +++ b/modules/interfaces/interfaceFeatures.py @@ -156,6 +156,7 @@ class FeatureInterface: featureCode: str, mandateId: str, label: str, + enabled: bool = True, copyTemplateRoles: bool = True ) -> FeatureInstance: """ @@ -171,6 +172,7 @@ class FeatureInterface: featureCode: Feature code (e.g., "trustee") mandateId: Mandate ID label: Instance label (e.g., "Buchhaltung 2025") + enabled: Whether the instance is enabled copyTemplateRoles: Whether to copy template roles Returns: @@ -182,7 +184,7 @@ class FeatureInterface: featureCode=featureCode, mandateId=mandateId, label=label, - enabled=True + enabled=enabled ) createdInstance = self.db.recordCreate(FeatureInstance, instance.model_dump()) @@ -382,6 +384,41 @@ class FeatureInterface: logger.error(f"Error syncing roles from template: {e}") raise ValueError(f"Failed to sync roles: {e}") + def updateFeatureInstance(self, instanceId: str, updateData: Dict[str, Any]) -> Optional[FeatureInstance]: + """ + Update a feature instance. + + Only label and enabled fields can be updated. + featureCode and mandateId are immutable. + + Args: + instanceId: FeatureInstance ID + updateData: Dictionary with fields to update (label, enabled) + + Returns: + Updated FeatureInstance object or None if not found + """ + try: + instance = self.getFeatureInstance(instanceId) + if not instance: + return None + + # Only allow updating specific fields + allowedFields = {"label", "enabled"} + filteredData = {k: v for k, v in updateData.items() if k in allowedFields} + + if not filteredData: + return instance + + updated = self.db.recordModify(FeatureInstance, instanceId, filteredData) + if updated: + cleanedRecord = {k: v for k, v in updated.items() if not k.startswith("_")} + return FeatureInstance(**cleanedRecord) + return None + except Exception as e: + logger.error(f"Error updating feature instance {instanceId}: {e}") + raise ValueError(f"Failed to update feature instance: {e}") + def deleteFeatureInstance(self, instanceId: str) -> bool: """ Delete a feature instance. diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index a99bfd0b..8ceefa6a 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -239,23 +239,40 @@ def buildRbacWhereClause( logger.warning(f"User {currentUser.id} has no mandateId for GROUP access") return {"condition": "1 = 0", "values": []} - # For UserInDB, filter by mandateId directly + # For UserInDB: Filter via UserMandate junction table + # Multi-Tenant Design: Users do NOT have mandateId - they are linked via UserMandate if table == "UserInDB": - return { - "condition": '"mandateId" = %s', - "values": [effectiveMandateId] - } - # For UserConnection, need to join with UserInDB or filter by mandateId in user - elif table == "UserConnection": - # Get all user IDs in the same mandate using direct SQL query try: with connector.connection.cursor() as cursor: + # Get all user IDs that are members of the current mandate cursor.execute( - 'SELECT "id" FROM "UserInDB" WHERE "mandateId" = %s', + 'SELECT "userId" FROM "UserMandate" WHERE "mandateId" = %s AND "enabled" = true', (effectiveMandateId,) ) - users = cursor.fetchall() - userIds = [u["id"] for u in users] + userMandates = cursor.fetchall() + userIds = [um["userId"] for um in userMandates] + if not userIds: + return {"condition": "1 = 0", "values": []} + placeholders = ",".join(["%s"] * len(userIds)) + return { + "condition": f'"id" IN ({placeholders})', + "values": userIds + } + except Exception as e: + logger.error(f"Error building GROUP filter for UserInDB via UserMandate: {e}") + return {"condition": "1 = 0", "values": []} + + # For UserConnection: Filter via UserMandate junction table + elif table == "UserConnection": + try: + with connector.connection.cursor() as cursor: + # Get all user IDs that are members of the current mandate + cursor.execute( + 'SELECT "userId" FROM "UserMandate" WHERE "mandateId" = %s AND "enabled" = true', + (effectiveMandateId,) + ) + userMandates = cursor.fetchall() + userIds = [um["userId"] for um in userMandates] if not userIds: return {"condition": "1 = 0", "values": []} placeholders = ",".join(["%s"] * len(userIds)) @@ -266,7 +283,8 @@ def buildRbacWhereClause( except Exception as e: logger.error(f"Error building GROUP filter for UserConnection: {e}") return {"condition": "1 = 0", "values": []} - # For other tables, filter by mandateId + + # For other tables, filter by mandateId field else: return { "condition": '"mandateId" = %s', diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 265a3a5c..2c8408e3 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -39,6 +39,7 @@ class FeatureInstanceCreate(BaseModel): """Request model for creating a feature instance""" featureCode: str = Field(..., description="Feature code (e.g., 'trustee', 'chatbot')") label: str = Field(..., description="Instance label (e.g., 'Buchhaltung 2025')") + enabled: bool = Field(True, description="Whether this feature instance is enabled") copyTemplateRoles: bool = Field(True, description="Whether to copy template roles on creation") @@ -64,7 +65,7 @@ class SyncRolesResult(BaseModel): @router.get("/", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listFeatures( +async def list_features( request: Request, context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: @@ -101,7 +102,7 @@ class FeaturesMyResponse(BaseModel): @router.get("/my", response_model=FeaturesMyResponse) @limiter.limit("60/minute") -async def getMyFeatureInstances( +async def get_my_feature_instances( request: Request, context: RequestContext = Depends(getRequestContext) ) -> FeaturesMyResponse: @@ -239,11 +240,12 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict permissions = { "tables": {}, "views": {}, - "fields": {} + "fields": {}, + "isAdmin": False # Flag if user has admin role } try: - from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole # Get FeatureAccess for this user and instance @@ -272,6 +274,15 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict logger.debug(f"_getInstancePermissions: No roles found for FeatureAccess {featureAccessId}") return permissions + # Check if user has admin role + for roleId in roleIds: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles: + roleLabel = roles[0].get("roleLabel", "").lower() + if "admin" in roleLabel: + permissions["isAdmin"] = True + break + # Get permissions (AccessRules) for all roles for roleId in roleIds: accessRules = rootInterface.db.getRecordset( @@ -323,6 +334,10 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict # item=None means all views - set a wildcard flag permissions["views"]["_all"] = True + # Derive view permissions from table permissions + # This allows UI navigation to be controlled by data access rights + _deriveViewPermissions(permissions) + return permissions except Exception as e: @@ -330,6 +345,51 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict return permissions +def _deriveViewPermissions(permissions: Dict[str, Any]) -> None: + """ + Derive UI view permissions from table/data permissions. + + Mapping: + - trustee-dashboard: always visible (basic access) + - trustee-positions: visible if READ on TrusteePosition + - trustee-documents: visible if READ on TrusteeDocument + - trustee-position-documents: visible if READ on TrusteePositionDocument + - trustee-instance-roles: visible only for admin roles + + This function modifies permissions["views"] in place. + """ + tables = permissions.get("tables", {}) + views = permissions.get("views", {}) + isAdmin = permissions.get("isAdmin", False) + + # If user has _all views permission, skip derivation + if views.get("_all"): + return + + # Dashboard is always visible for users with any access + if "trustee-dashboard" not in views: + views["trustee-dashboard"] = True + + # Positions view: requires READ on TrusteePosition + if "trustee-positions" not in views: + positionPerms = tables.get("TrusteePosition", {}) + views["trustee-positions"] = positionPerms.get("read", "n") != "n" + + # Documents view: requires READ on TrusteeDocument + if "trustee-documents" not in views: + documentPerms = tables.get("TrusteeDocument", {}) + views["trustee-documents"] = documentPerms.get("read", "n") != "n" + + # Position-Documents view: requires READ on TrusteePositionDocument + if "trustee-position-documents" not in views: + linkPerms = tables.get("TrusteePositionDocument", {}) + views["trustee-position-documents"] = linkPerms.get("read", "n") != "n" + + # Instance-roles (admin) view: requires admin role + if "trustee-instance-roles" not in views: + views["trustee-instance-roles"] = isAdmin + + def _mergeAccessLevel(current: str, new: str) -> str: """Merge two access levels, returning the highest.""" levels = {"n": 0, "m": 1, "g": 2, "a": 3} @@ -343,7 +403,7 @@ def _mergeAccessLevel(current: str, new: str) -> str: @router.post("/", response_model=Dict[str, Any]) @limiter.limit("10/minute") -async def createFeature( +async def create_feature( request: Request, code: str = Query(..., description="Unique feature code"), label: Dict[str, str] = None, @@ -397,7 +457,7 @@ async def createFeature( @router.get("/instances", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listFeatureInstances( +async def list_feature_instances( request: Request, featureCode: Optional[str] = Query(None, description="Filter by feature code"), context: RequestContext = Depends(getRequestContext) @@ -439,7 +499,7 @@ async def listFeatureInstances( @router.get("/instances/{instanceId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def getFeatureInstance( +async def get_feature_instance( request: Request, instanceId: str, context: RequestContext = Depends(getRequestContext) @@ -483,7 +543,7 @@ async def getFeatureInstance( @router.post("/instances", response_model=Dict[str, Any]) @limiter.limit("10/minute") -async def createFeatureInstance( +async def create_feature_instance( request: Request, data: FeatureInstanceCreate, context: RequestContext = Depends(getRequestContext) @@ -525,6 +585,7 @@ async def createFeatureInstance( featureCode=data.featureCode, mandateId=str(context.mandateId), label=data.label, + enabled=data.enabled, copyTemplateRoles=data.copyTemplateRoles ) @@ -547,7 +608,7 @@ async def createFeatureInstance( @router.delete("/instances/{instanceId}", response_model=Dict[str, str]) @limiter.limit("10/minute") -async def deleteFeatureInstance( +async def delete_feature_instance( request: Request, instanceId: str, context: RequestContext = Depends(getRequestContext) @@ -603,9 +664,90 @@ async def deleteFeatureInstance( ) +class FeatureInstanceUpdate(BaseModel): + """Request model for updating a feature instance.""" + label: Optional[str] = Field(None, description="New label for the instance") + enabled: Optional[bool] = Field(None, description="Enable/disable the instance") + + +@router.put("/instances/{instanceId}", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def updateFeatureInstance( + request: Request, + instanceId: str, + data: FeatureInstanceUpdate, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Update a feature instance (label, enabled). + + Requires Mandate-Admin role. + + Args: + instanceId: FeatureInstance ID + data: Fields to update (label, enabled) + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check mandate admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate-Admin role required to update feature instances" + ) + + # Build update data (only non-None values) + updateData = {} + if data.label is not None: + updateData["label"] = data.label + if data.enabled is not None: + updateData["enabled"] = data.enabled + + if not updateData: + return instance.model_dump() + + updated = featureInterface.updateFeatureInstance(instanceId, updateData) + if not updated: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to update feature instance" + ) + + logger.info(f"User {context.user.id} updated feature instance {instanceId}: {updateData}") + + return updated.model_dump() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating feature instance {instanceId}: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update feature instance: {str(e)}" + ) + + @router.post("/instances/{instanceId}/sync-roles", response_model=SyncRolesResult) @limiter.limit("10/minute") -async def syncInstanceRoles( +async def sync_instance_roles( request: Request, instanceId: str, addOnly: bool = Query(True, description="Only add missing roles, don't remove extras"), @@ -672,7 +814,7 @@ async def syncInstanceRoles( @router.get("/templates/roles", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listTemplateRoles( +async def list_template_roles( request: Request, featureCode: Optional[str] = Query(None, description="Filter by feature code"), sysAdmin: User = Depends(requireSysAdmin) @@ -702,7 +844,7 @@ async def listTemplateRoles( @router.post("/templates/roles", response_model=Dict[str, Any]) @limiter.limit("10/minute") -async def createTemplateRole( +async def create_template_role( request: Request, roleLabel: str = Query(..., description="Role label (e.g., 'admin', 'viewer')"), featureCode: str = Query(..., description="Feature code this role belongs to"), @@ -780,7 +922,7 @@ class FeatureInstanceUserResponse(BaseModel): @router.get("/instances/{instanceId}/users", response_model=List[FeatureInstanceUserResponse]) @limiter.limit("60/minute") -async def listFeatureInstanceUsers( +async def list_feature_instance_users( request: Request, instanceId: str, context: RequestContext = Depends(getRequestContext) @@ -872,7 +1014,7 @@ async def listFeatureInstanceUsers( @router.post("/instances/{instanceId}/users", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def addUserToFeatureInstance( +async def add_user_to_feature_instance( request: Request, instanceId: str, data: FeatureInstanceUserCreate, @@ -976,7 +1118,7 @@ async def addUserToFeatureInstance( @router.delete("/instances/{instanceId}/users/{userId}", response_model=Dict[str, str]) @limiter.limit("30/minute") -async def removeUserFromFeatureInstance( +async def remove_user_from_feature_instance( request: Request, instanceId: str, userId: str, @@ -1057,7 +1199,7 @@ async def removeUserFromFeatureInstance( @router.put("/instances/{instanceId}/users/{userId}/roles", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def updateFeatureInstanceUserRoles( +async def update_feature_instance_user_roles( request: Request, instanceId: str, userId: str, @@ -1154,7 +1296,7 @@ async def updateFeatureInstanceUserRoles( @router.get("/instances/{instanceId}/available-roles", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getFeatureInstanceAvailableRoles( +async def get_feature_instance_available_roles( request: Request, instanceId: str, context: RequestContext = Depends(getRequestContext) @@ -1222,7 +1364,7 @@ async def getFeatureInstanceAvailableRoles( @router.get("/{featureCode}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def getFeature( +async def get_feature( request: Request, featureCode: str, context: RequestContext = Depends(getRequestContext) diff --git a/modules/routes/routeAdminRbacExport.py b/modules/routes/routeAdminRbacExport.py index 11932f18..c44c3b6b 100644 --- a/modules/routes/routeAdminRbacExport.py +++ b/modules/routes/routeAdminRbacExport.py @@ -72,7 +72,7 @@ class RbacImportResult(BaseModel): @router.get("/export/global", response_model=RbacExportData) @limiter.limit("10/minute") -async def exportGlobalRbac( +async def export_global_rbac( request: Request, sysAdmin: User = Depends(requireSysAdmin) ) -> RbacExportData: @@ -138,7 +138,7 @@ async def exportGlobalRbac( @router.post("/import/global", response_model=RbacImportResult) @limiter.limit("5/minute") -async def importGlobalRbac( +async def import_global_rbac( request: Request, file: UploadFile = File(..., description="JSON file with RBAC export data"), updateExisting: bool = False, @@ -285,7 +285,7 @@ async def importGlobalRbac( @router.get("/export/mandate", response_model=RbacExportData) @limiter.limit("10/minute") -async def exportMandateRbac( +async def export_mandate_rbac( request: Request, includeFeatureInstances: bool = True, context: RequestContext = Depends(getRequestContext) @@ -380,7 +380,7 @@ async def exportMandateRbac( @router.post("/import/mandate", response_model=RbacImportResult) @limiter.limit("5/minute") -async def importMandateRbac( +async def import_mandate_rbac( request: Request, file: UploadFile = File(..., description="JSON file with RBAC export data"), updateExisting: bool = False, diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index 70d0beaa..ad8a0de5 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -76,7 +76,7 @@ router = APIRouter( @router.get("/", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listRoles( +async def list_roles( request: Request, currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: @@ -121,7 +121,7 @@ async def listRoles( @router.get("/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getRoleOptions( +async def get_role_options( request: Request, currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: @@ -162,7 +162,7 @@ async def getRoleOptions( @router.post("/", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def createRole( +async def create_role( request: Request, role: Role = Body(...), currentUser: User = Depends(requireSysAdmin) @@ -206,7 +206,7 @@ async def createRole( @router.get("/{roleId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def getRole( +async def get_role( request: Request, roleId: str = Path(..., description="Role ID"), currentUser: User = Depends(requireSysAdmin) @@ -250,7 +250,7 @@ async def getRole( @router.put("/{roleId}", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def updateRole( +async def update_role( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), @@ -298,7 +298,7 @@ async def updateRole( @router.delete("/{roleId}", response_model=Dict[str, str]) @limiter.limit("30/minute") -async def deleteRole( +async def delete_role( request: Request, roleId: str = Path(..., description="Role ID"), currentUser: User = Depends(requireSysAdmin) @@ -342,7 +342,7 @@ async def deleteRole( @router.get("/users", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listUsersWithRoles( +async def list_users_with_roles( request: Request, roleLabel: Optional[str] = Query(None, description="Filter by role label"), mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), @@ -412,7 +412,7 @@ async def listUsersWithRoles( @router.get("/users/{userId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def getUserRoles( +async def get_user_roles( request: Request, userId: str = Path(..., description="User ID"), currentUser: User = Depends(requireSysAdmin) @@ -462,7 +462,7 @@ async def getUserRoles( @router.put("/users/{userId}/roles", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def updateUserRoles( +async def update_user_roles( request: Request, userId: str = Path(..., description="User ID"), newRoleLabels: List[str] = Body(..., description="List of role labels to assign"), @@ -559,7 +559,7 @@ async def updateUserRoles( @router.post("/users/{userId}/roles/{roleLabel}", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def addUserRole( +async def add_user_role( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to add"), @@ -641,7 +641,7 @@ async def addUserRole( @router.delete("/users/{userId}/roles/{roleLabel}", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def removeUserRole( +async def remove_user_role( request: Request, userId: str = Path(..., description="User ID"), roleLabel: str = Path(..., description="Role label to remove"), @@ -721,7 +721,7 @@ async def removeUserRole( @router.get("/roles/{roleLabel}/users", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getUsersWithRole( +async def get_users_with_role( request: Request, roleLabel: str = Path(..., description="Role label"), mandateId: Optional[str] = Query(None, description="Filter by mandate ID (via UserMandate)"), diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index dcd32bbc..a1990543 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -34,7 +34,7 @@ router = APIRouter( @router.get("/permissions", response_model=UserPermissions) @limiter.limit("300/minute") # Raised from 60 - sidebar checks many pages individually -async def getPermissions( +async def get_permissions( request: Request, context: str = Query(..., description="Context type: DATA, UI, or RESOURCE"), item: Optional[str] = Query(None, description="Item identifier (table name, UI path, or resource path)"), @@ -98,7 +98,7 @@ async def getPermissions( @router.get("/permissions/all", response_model=Dict[str, Any]) @limiter.limit("120/minute") # Raised from 30 - optimized endpoint for bulk permission fetch -async def getAllPermissions( +async def get_all_permissions( request: Request, context: Optional[str] = Query(None, description="Context type: UI or RESOURCE (if not provided, returns both)"), reqContext: RequestContext = Depends(getRequestContext) @@ -224,7 +224,7 @@ async def getAllPermissions( @router.get("/rules", response_model=PaginatedResponse) @limiter.limit("30/minute") -async def getAccessRules( +async def get_access_rules( request: Request, roleLabel: Optional[str] = Query(None, description="Filter by role label"), context: Optional[str] = Query(None, description="Filter by context (DATA, UI, RESOURCE)"), @@ -313,7 +313,7 @@ async def getAccessRules( @router.get("/rules/by-role/{roleId}", response_model=PaginatedResponse) @limiter.limit("30/minute") -async def getAccessRulesByRole( +async def get_access_rules_by_role( request: Request, roleId: str = Path(..., description="Role ID to get rules for"), currentUser: User = Depends(requireSysAdmin) @@ -357,7 +357,7 @@ async def getAccessRulesByRole( @router.get("/rules/{ruleId}", response_model=dict) @limiter.limit("30/minute") -async def getAccessRule( +async def get_access_rule( request: Request, ruleId: str = Path(..., description="Access rule ID"), currentUser: User = Depends(requireSysAdmin) @@ -399,7 +399,7 @@ async def getAccessRule( @router.post("/rules", response_model=dict) @limiter.limit("30/minute") -async def createAccessRule( +async def create_access_rule( request: Request, accessRuleData: dict = Body(..., description="Access rule data"), currentUser: User = Depends(requireSysAdmin) @@ -465,7 +465,7 @@ async def createAccessRule( @router.put("/rules/{ruleId}", response_model=dict) @limiter.limit("30/minute") -async def updateAccessRule( +async def update_access_rule( request: Request, ruleId: str = Path(..., description="Access rule ID"), accessRuleData: dict = Body(..., description="Updated access rule data"), @@ -548,7 +548,7 @@ async def updateAccessRule( @router.delete("/rules/{ruleId}") @limiter.limit("30/minute") -async def deleteAccessRule( +async def delete_access_rule( request: Request, ruleId: str = Path(..., description="Access rule ID"), currentUser: User = Depends(requireSysAdmin) @@ -606,7 +606,7 @@ async def deleteAccessRule( @router.get("/roles", response_model=PaginatedResponse) @limiter.limit("60/minute") -async def listRoles( +async def list_roles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), includeTemplates: bool = Query(False, description="Include feature template roles"), @@ -775,7 +775,7 @@ async def listRoles( @router.get("/roles/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getRoleOptions( +async def get_role_options( request: Request, currentUser: User = Depends(requireSysAdmin) ) -> List[Dict[str, Any]]: @@ -816,7 +816,7 @@ async def getRoleOptions( @router.post("/roles", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def createRole( +async def create_role( request: Request, role: Role = Body(...), currentUser: User = Depends(requireSysAdmin) @@ -865,7 +865,7 @@ async def createRole( @router.get("/roles/{roleId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def getRole( +async def get_role( request: Request, roleId: str = Path(..., description="Role ID"), currentUser: User = Depends(requireSysAdmin) @@ -912,7 +912,7 @@ async def getRole( @router.put("/roles/{roleId}", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def updateRole( +async def update_role( request: Request, roleId: str = Path(..., description="Role ID"), role: Role = Body(...), @@ -965,7 +965,7 @@ async def updateRole( @router.delete("/roles/{roleId}", response_model=Dict[str, str]) @limiter.limit("30/minute") -async def deleteRole( +async def delete_role( request: Request, roleId: str = Path(..., description="Role ID"), currentUser: User = Depends(requireSysAdmin) diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 2dade569..37200186 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -100,7 +100,7 @@ router = APIRouter( @router.get("/statuses/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getConnectionStatusOptions( +async def get_connection_status_options( request: Request, currentUser: User = Depends(getCurrentUser) ) -> List[Dict[str, Any]]: @@ -116,7 +116,7 @@ async def getConnectionStatusOptions( @router.get("/authorities/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getAuthAuthorityOptions( +async def get_auth_authority_options( request: Request, currentUser: User = Depends(getCurrentUser) ) -> List[Dict[str, Any]]: diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 37c871ab..23358947 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -324,7 +324,7 @@ async def delete_mandate( @router.get("/{targetMandateId}/users") @limiter.limit("60/minute") -async def listMandateUsers( +async def list_mandate_users( request: Request, targetMandateId: str = Path(..., description="ID of the mandate"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), @@ -486,7 +486,7 @@ async def listMandateUsers( @router.post("/{targetMandateId}/users", response_model=UserMandateResponse) @limiter.limit("30/minute") -async def addUserToMandate( +async def add_user_to_mandate( request: Request, targetMandateId: str = Path(..., description="ID of the mandate"), data: UserMandateCreate = Body(...), @@ -602,7 +602,7 @@ async def addUserToMandate( @router.delete("/{targetMandateId}/users/{targetUserId}", response_model=Dict[str, str]) @limiter.limit("30/minute") -async def removeUserFromMandate( +async def remove_user_from_mandate( request: Request, targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user to remove"), @@ -680,7 +680,7 @@ async def removeUserFromMandate( @router.put("/{targetMandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) @limiter.limit("30/minute") -async def updateUserRolesInMandate( +async def update_user_roles_in_mandate( request: Request, targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user"), diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index b3da0c2e..f963a33c 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -152,7 +152,7 @@ router = APIRouter( @router.get("/options", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def getUserOptions( +async def get_user_options( request: Request, context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: @@ -631,7 +631,7 @@ async def change_password( @router.post("/{userId}/send-password-link") @limiter.limit("10/minute") -async def sendPasswordLink( +async def send_password_link( request: Request, userId: str = Path(..., description="ID of the user to send password setup link"), frontendUrl: str = Body(..., embed=True), diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index 4f5f2aa4..c0b219ec 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -73,7 +73,7 @@ class DeletionResult(BaseModel): @router.get("/data-export", response_model=DataExportResponse) @limiter.limit("5/minute") -async def exportUserData( +async def export_user_data( request: Request, currentUser: User = Depends(getCurrentUser) ) -> DataExportResponse: @@ -238,7 +238,7 @@ async def exportUserData( @router.get("/data-portability") @limiter.limit("5/minute") -async def exportPortableData( +async def export_portable_data( request: Request, currentUser: User = Depends(getCurrentUser) ) -> JSONResponse: @@ -333,7 +333,7 @@ async def exportPortableData( @router.delete("/", response_model=DeletionResult) @limiter.limit("1/hour") -async def deleteAccount( +async def delete_account( request: Request, confirmDeletion: bool = False, currentUser: User = Depends(getCurrentUser) @@ -465,7 +465,7 @@ async def deleteAccount( @router.get("/consent-info", response_model=Dict[str, Any]) @limiter.limit("30/minute") -async def getConsentInfo( +async def get_consent_info( request: Request, currentUser: User = Depends(getCurrentUser) ) -> Dict[str, Any]: diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index 3b059f9c..0e0259eb 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -108,7 +108,7 @@ class RegisterAndAcceptResponse(BaseModel): @router.post("/", response_model=InvitationResponse) @limiter.limit("30/minute") -async def createInvitation( +async def create_invitation( request: Request, data: InvitationCreate, context: RequestContext = Depends(getRequestContext) @@ -234,7 +234,7 @@ async def createInvitation( @router.get("/", response_model=List[Dict[str, Any]]) @limiter.limit("60/minute") -async def listInvitations( +async def list_invitations( request: Request, includeUsed: bool = Query(False, description="Include already used invitations"), includeExpired: bool = Query(False, description="Include expired invitations"), @@ -313,7 +313,7 @@ async def listInvitations( @router.delete("/{invitationId}", response_model=Dict[str, str]) @limiter.limit("30/minute") -async def revokeInvitation( +async def revoke_invitation( request: Request, invitationId: str, context: RequestContext = Depends(getRequestContext) @@ -397,7 +397,7 @@ async def revokeInvitation( @router.get("/validate/{token}", response_model=InvitationValidation) @limiter.limit("30/minute") -async def validateInvitation( +async def validate_invitation( request: Request, token: str ) -> InvitationValidation: @@ -482,7 +482,7 @@ async def validateInvitation( @router.post("/accept/{token}", response_model=Dict[str, Any]) @limiter.limit("10/minute") -async def acceptInvitation( +async def accept_invitation( request: Request, token: str, currentUser: User = Depends(getCurrentUser) @@ -614,7 +614,7 @@ async def acceptInvitation( @router.post("/register-and-accept", response_model=RegisterAndAcceptResponse) @limiter.limit("10/minute") # Stricter rate limit for registration -async def registerAndAcceptInvitation( +async def register_and_accept_invitation( request: Request, data: RegisterAndAcceptRequest ) -> RegisterAndAcceptResponse: diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index 953dd5f2..753fb16f 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -38,7 +38,7 @@ router = APIRouter( @router.get("/subscriptions", response_model=PaginatedResponse[MessagingSubscription]) @limiter.limit("60/minute") -async def getSubscriptions( +async def get_subscriptions( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), currentUser: User = Depends(getCurrentUser) @@ -79,7 +79,7 @@ async def getSubscriptions( @router.post("/subscriptions", response_model=MessagingSubscription) @limiter.limit("60/minute") -async def createSubscription( +async def create_subscription( request: Request, subscription: MessagingSubscription, currentUser: User = Depends(getCurrentUser) @@ -95,7 +95,7 @@ async def createSubscription( @router.get("/subscriptions/{subscriptionId}", response_model=MessagingSubscription) @limiter.limit("60/minute") -async def getSubscription( +async def get_subscription( request: Request, subscriptionId: str = Path(..., description="ID of the subscription"), currentUser: User = Depends(getCurrentUser) @@ -115,7 +115,7 @@ async def getSubscription( @router.put("/subscriptions/{subscriptionId}", response_model=MessagingSubscription) @limiter.limit("60/minute") -async def updateSubscription( +async def update_subscription( request: Request, subscriptionId: str = Path(..., description="ID of the subscription to update"), subscriptionData: MessagingSubscription = Body(...), @@ -145,7 +145,7 @@ async def updateSubscription( @router.delete("/subscriptions/{subscriptionId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def deleteSubscription( +async def delete_subscription( request: Request, subscriptionId: str = Path(..., description="ID of the subscription to delete"), currentUser: User = Depends(getCurrentUser) @@ -174,7 +174,7 @@ async def deleteSubscription( @router.get("/subscriptions/{subscriptionId}/registrations", response_model=PaginatedResponse[MessagingSubscriptionRegistration]) @limiter.limit("60/minute") -async def getSubscriptionRegistrations( +async def get_subscription_registrations( request: Request, subscriptionId: str = Path(..., description="ID of the subscription"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), @@ -219,7 +219,7 @@ async def getSubscriptionRegistrations( @router.post("/subscriptions/{subscriptionId}/subscribe", response_model=MessagingSubscriptionRegistration) @limiter.limit("60/minute") -async def subscribeUser( +async def subscribe_user( request: Request, subscriptionId: str = Path(..., description="ID of the subscription"), channel: MessagingChannel = Body(..., embed=True), @@ -241,7 +241,7 @@ async def subscribeUser( @router.delete("/subscriptions/{subscriptionId}/unsubscribe", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def unsubscribeUser( +async def unsubscribe_user( request: Request, subscriptionId: str = Path(..., description="ID of the subscription"), channel: MessagingChannel = Body(..., embed=True), @@ -267,7 +267,7 @@ async def unsubscribeUser( @router.get("/registrations", response_model=PaginatedResponse[MessagingSubscriptionRegistration]) @limiter.limit("60/minute") -async def getMyRegistrations( +async def get_my_registrations( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), currentUser: User = Depends(getCurrentUser) @@ -311,7 +311,7 @@ async def getMyRegistrations( @router.put("/registrations/{registrationId}", response_model=MessagingSubscriptionRegistration) @limiter.limit("60/minute") -async def updateRegistration( +async def update_registration( request: Request, registrationId: str = Path(..., description="ID of the registration to update"), registrationData: MessagingSubscriptionRegistration = Body(...), @@ -341,7 +341,7 @@ async def updateRegistration( @router.delete("/registrations/{registrationId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") -async def deleteRegistration( +async def delete_registration( request: Request, registrationId: str = Path(..., description="ID of the registration to delete"), currentUser: User = Depends(getCurrentUser) @@ -376,7 +376,7 @@ def _getTriggerKey(request: Request) -> str: @router.post("/trigger/{subscriptionId}", response_model=MessagingSubscriptionExecutionResult) @limiter.limit("60/minute", key_func=_getTriggerKey) -async def triggerSubscription( +async def trigger_subscription( request: Request, subscriptionId: str = Path(..., description="ID of the subscription to trigger"), eventParameters: Dict[str, Any] = Body(...), @@ -440,7 +440,7 @@ def _hasTriggerPermission(context: RequestContext) -> bool: @router.get("/deliveries", response_model=PaginatedResponse[MessagingDelivery]) @limiter.limit("60/minute") -async def getDeliveries( +async def get_deliveries( request: Request, subscriptionId: Optional[str] = Query(None, description="Filter by subscription ID"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), @@ -486,7 +486,7 @@ async def getDeliveries( @router.get("/deliveries/{deliveryId}", response_model=MessagingDelivery) @limiter.limit("60/minute") -async def getDelivery( +async def get_delivery( request: Request, deliveryId: str = Path(..., description="ID of the delivery"), currentUser: User = Depends(getCurrentUser) diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 4c61cbc1..8ab211cf 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -106,18 +106,9 @@ async def login( # Get gateway interface with root privileges for authentication rootInterface = getRootInterface() - # Get default mandate ID - defaultMandateId = rootInterface.getInitialId(Mandate) - if not defaultMandateId: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No default mandate found" - ) - - # Set the mandate ID on the interface - rootInterface.mandateId = defaultMandateId - # Authenticate user + # Note: authenticateLocalUser uses _getUserForAuthentication which bypasses RBAC + # This is correct because users are mandate-independent (Multi-Tenant Design) user = rootInterface.authenticateLocalUser( username=formData.username, password=formData.password @@ -265,16 +256,9 @@ async def register_user( # Get gateway interface with root privileges since this is a public endpoint appInterface = getRootInterface() - # Get default mandate ID - defaultMandateId = appInterface.getInitialId(Mandate) - if not defaultMandateId: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No default mandate found" - ) - - # Set the mandate ID on the interface - appInterface.mandateId = defaultMandateId + # Note: User registration does NOT require mandateId context + # Users are mandate-independent (Multi-Tenant Design) + # Mandate assignment happens via createUserMandate() after registration # Frontend URL is required - no fallback baseUrl = frontendUrl.rstrip("/") @@ -548,7 +532,7 @@ async def check_username_availability( @router.post("/password-reset-request") @limiter.limit("5/minute") -async def passwordResetRequest( +async def password_reset_request( request: Request, username: str = Body(..., embed=True), frontendUrl: str = Body(..., embed=True) @@ -628,7 +612,7 @@ Falls Sie diese Anforderung nicht gestellt haben, können Sie diese E-Mail ignor @router.post("/password-reset") @limiter.limit("10/minute") -async def passwordReset( +async def password_reset( request: Request, token: str = Body(..., embed=True), password: str = Body(..., embed=True) diff --git a/modules/routes/routeVoiceGoogle.py b/modules/routes/routeVoiceGoogle.py index 401bfc0b..8e72207c 100644 --- a/modules/routes/routeVoiceGoogle.py +++ b/modules/routes/routeVoiceGoogle.py @@ -39,7 +39,7 @@ class ConnectionManager: del activeConnections[connectionId] logger.info(f"WebSocket disconnected: {connectionId}") - async def sendPersonalMessage(self, message: dict, websocket: WebSocket): + async def send_personal_message(self, message: dict, websocket: WebSocket): try: await websocket.send_text(json.dumps(message)) except Exception as e: diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index a80be79c..d8b5c078 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -28,8 +28,11 @@ class MethodAiOperationsTester: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) self.services = None self.methodAi = None @@ -119,7 +122,7 @@ class MethodAiOperationsTester: currentAction=0, totalTasks=0, totalActions=0, - mandateId=self.testUser.mandateId, + mandateId=self.testMandateId, messageIds=[], workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, maxSteps=5 @@ -277,7 +280,7 @@ class MethodAiOperationsTester: currentAction=0, totalTasks=0, totalActions=0, - mandateId=self.testUser.mandateId, + mandateId=self.testMandateId, messageIds=[], workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, maxSteps=5 diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 657946a9..7b5d6d73 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -28,8 +28,11 @@ class AIBehaviorTester: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -60,7 +63,7 @@ class AIBehaviorTester: currentAction=0, totalTasks=0, totalActions=0, - mandateId=self.testUser.mandateId, + mandateId=self.testMandateId, messageIds=[], workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, maxSteps=5 diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index fac1ab41..960c7d8d 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -30,8 +30,11 @@ class WorkflowWithDocumentsTester: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -45,7 +48,7 @@ class WorkflowWithDocumentsTester: logging.getLogger().setLevel(logging.INFO) print(f"Initialized test with user: {self.testUser.id}") - print(f"Mandate ID: {self.testUser.mandateId}") + print(f"Test Mandate ID: {self.testMandateId}") def createCsvTemplate(self) -> str: """Create a CSV template file for prime numbers.""" diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 4b39454a..121b22a1 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -32,8 +32,11 @@ class WorkflowPromptVariationsTester: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -46,7 +49,7 @@ class WorkflowPromptVariationsTester: logging.getLogger().setLevel(logging.INFO) print(f"Initialized test with user: {self.testUser.id}") - print(f"Mandate ID: {self.testUser.mandateId}") + print(f"Test Mandate ID: {self.testMandateId}") def _createFile(self, fileName: str, mimeType: str, content: str) -> str: """Helper method to create a file and return its ID.""" diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index a6f99236..e646bc99 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -31,8 +31,11 @@ class DocumentGenerationFormatsTester: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -52,7 +55,7 @@ class DocumentGenerationFormatsTester: logging.getLogger().setLevel(logging.INFO) print(f"Initialized test with user: {self.testUser.id}") - print(f"Mandate ID: {self.testUser.mandateId}") + print(f"Test Mandate ID: {self.testMandateId}") print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") # Upload PDF file for testing diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index e1990910..c9542efe 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -31,8 +31,11 @@ class DocumentGenerationFormatsTester10: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -52,7 +55,7 @@ class DocumentGenerationFormatsTester10: logging.getLogger().setLevel(logging.INFO) print(f"Initialized test with user: {self.testUser.id}") - print(f"Mandate ID: {self.testUser.mandateId}") + print(f"Test Mandate ID: {self.testMandateId}") print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") # Upload PDF file for testing diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 43c294e4..76f26a61 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -33,8 +33,11 @@ class CodeGenerationFormatsTester11: def __init__(self): # Use root user for testing (has full access to everything) from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelUam import Mandate rootInterface = getRootInterface() self.testUser = rootInterface.currentUser + # Get initial mandate ID for testing (User has no mandateId - use initial mandate) + self.testMandateId = rootInterface.getInitialId(Mandate) # Initialize services using the existing system self.services = getServices(self.testUser, None) # Test user, no workflow @@ -53,7 +56,7 @@ class CodeGenerationFormatsTester11: logging.getLogger().setLevel(logging.INFO) print(f"Initialized test with user: {self.testUser.id}") - print(f"Mandate ID: {self.testUser.mandateId}") + print(f"Test Mandate ID: {self.testMandateId}") print(f"Debug logging enabled: {APP_CONFIG.get('APP_DEBUG_CHAT_WORKFLOW_ENABLED', False)}") def createTestPrompt(self, format: str) -> str: From 2fc80342600022c5209c5766d8b0afc332a07a8f Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 25 Jan 2026 03:01:01 +0100 Subject: [PATCH 23/32] rbac rules tested and fixed --- app.py | 15 +- modules/auth/authentication.py | 76 +-- modules/features/aichat/mainAiChat.py | 166 ------ modules/features/automation/mainAutomation.py | 27 +- .../chatbot/interfaceFeatureChatbot.py | 4 +- modules/features/chatbot/mainChatbot.py | 20 +- .../datamodelFeatureNeutralizer.py | 0 .../interfaceFeatureNeutralizer.py | 2 +- .../mainNeutralizePlayground.py | 0 .../mainNeutralizer.py | 29 +- .../routeFeatureNeutralizer.py | 0 .../mainServiceNeutralization.py | 4 +- .../serviceNeutralization/subParseString.py | 0 .../serviceNeutralization/subPatterns.py | 0 .../serviceNeutralization/subProcessBinary.py | 0 .../serviceNeutralization/subProcessCommon.py | 0 .../serviceNeutralization/subProcessList.py | 0 .../serviceNeutralization/subProcessText.py | 0 .../realEstate/interfaceFeatureRealEstate.py | 4 +- modules/features/realEstate/mainRealEstate.py | 34 +- .../trustee/interfaceFeatureTrustee.py | 99 +++- modules/features/trustee/mainTrustee.py | 69 ++- .../features/trustee/routeFeatureTrustee.py | 66 ++- modules/interfaces/interfaceBootstrap.py | 159 +++++- modules/interfaces/interfaceDbApp.py | 3 +- modules/interfaces/interfaceDbChat.py | 4 +- modules/interfaces/interfaceDbManagement.py | 4 +- modules/interfaces/interfaceRbac.py | 102 +++- modules/routes/routeAdminFeatures.py | 52 +- modules/routes/routeAdminRbacRules.py | 219 +++++++- modules/security/rbacCatalog.py | 45 +- modules/system/__init__.py | 14 + modules/system/mainSystem.py | 423 ++++++++++++++ .../featureRegistry.py => system/registry.py} | 24 +- modules/system/routeSystem.py | 515 ++++++++++++++++++ ...cript_db_migrate_accessrules_objectkeys.py | 183 +++++++ scripts/script_export_accessrules.py | 96 ++++ 37 files changed, 2081 insertions(+), 377 deletions(-) delete mode 100644 modules/features/aichat/mainAiChat.py rename modules/features/{neutralizer => neutralization}/datamodelFeatureNeutralizer.py (100%) rename modules/features/{neutralizer => neutralization}/interfaceFeatureNeutralizer.py (98%) rename modules/features/{neutralizer => neutralization}/mainNeutralizePlayground.py (100%) rename modules/features/{neutralizer => neutralization}/mainNeutralizer.py (76%) rename modules/features/{neutralizer => neutralization}/routeFeatureNeutralizer.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/mainServiceNeutralization.py (98%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subParseString.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subPatterns.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subProcessBinary.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subProcessCommon.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subProcessList.py (100%) rename modules/features/{neutralizer => neutralization}/serviceNeutralization/subProcessText.py (100%) create mode 100644 modules/system/__init__.py create mode 100644 modules/system/mainSystem.py rename modules/{features/featureRegistry.py => system/registry.py} (82%) create mode 100644 modules/system/routeSystem.py create mode 100644 scripts/script_db_migrate_accessrules_objectkeys.py create mode 100644 scripts/script_export_accessrules.py diff --git a/app.py b/app.py index a73cbcbc..b78e5d8b 100644 --- a/app.py +++ b/app.py @@ -21,7 +21,7 @@ from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager from modules.workflows.automation import subAutomationSchedule from modules.interfaces.interfaceDbApp import getRootInterface -from modules.features.featureRegistry import loadFeatureMainModules +from modules.system.registry import loadFeatureMainModules class DailyRotatingFileHandler(RotatingFileHandler): """ @@ -346,8 +346,8 @@ def _generateOperationId(route) -> str: # START APP app = FastAPI( - title="PowerOn | Data Platform API", - description=f"Backend API for the Multi-Agent Platform by ValueOn AG ({instanceLabel})", + title="PowerOn AG | Workflow Engine", + description=f"API for dynamic SaaS platforms ({instanceLabel})", lifespan=lifespan, swagger_ui_init_oauth={ "usePkceWithAuthorizationCodeGrant": True, @@ -501,11 +501,18 @@ app.include_router(gdprRouter) from modules.routes.routeChat import router as chatRouter app.include_router(chatRouter) +# ============================================================================ +# SYSTEM ROUTES (Navigation, etc.) +# ============================================================================ +from modules.system.routeSystem import router as systemRouter, navigationRouter +app.include_router(systemRouter) +app.include_router(navigationRouter) + # ============================================================================ # PLUG&PLAY FEATURE ROUTERS # Dynamically load routers from feature containers in modules/features/ # ============================================================================ -from modules.features.featureRegistry import loadFeatureRouters +from modules.system.registry import loadFeatureRouters featureLoadResults = loadFeatureRouters(app) logger.info(f"Feature router load results: {featureLoadResults}") diff --git a/modules/auth/authentication.py b/modules/auth/authentication.py index c6eaafad..8c918e0f 100644 --- a/modules/auth/authentication.py +++ b/modules/auth/authentication.py @@ -276,8 +276,11 @@ def getRequestContext( Determines request context from headers. Checks authorization and loads role IDs. - IMPORTANT: Even SysAdmin needs explicit membership for mandate context! - SysAdmin flag does NOT give implicit access to mandate data. + Security Model: + - Regular users: Must be explicit members of mandates/feature instances + - SysAdmin users: Can access ANY mandate for administrative operations, + but don't get implicit roleIds (no automatic data access rights). + Routes can check ctx.isSysAdmin to allow admin operations. Args: request: FastAPI Request object @@ -289,57 +292,66 @@ def getRequestContext( RequestContext with user, mandate, roles Raises: - HTTPException 403: If user is not member of mandate or has no feature access + HTTPException 403: If non-SysAdmin user is not member of mandate or has no feature access """ ctx = RequestContext(user=currentUser) + isSysAdmin = getattr(currentUser, 'isSysAdmin', False) # Get root interface for membership checks rootInterface = getRootInterface() if mandateId: - # Check mandate membership - ALSO for SysAdmin! - # SysAdmin must be explicitly added to the mandate + # Check mandate membership membership = rootInterface.getUserMandate(currentUser.id, mandateId) - if not membership: - # No implicit access for SysAdmin - Fail-Fast! + + if membership: + # User is a member - load their roles + if not membership.enabled: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Mandate membership is disabled" + ) + ctx.mandateId = mandateId + ctx.roleIds = rootInterface.getRoleIdsForUserMandate(membership.id) + elif isSysAdmin: + # SysAdmin can access any mandate for admin operations + # But they don't get roleIds - no implicit data access + ctx.mandateId = mandateId + # roleIds stays empty - SysAdmin must rely on isSysAdmin flag for authorization + logger.debug(f"SysAdmin {currentUser.id} accessing mandate {mandateId} without membership") + else: + # Regular user without membership - denied logger.warning(f"User {currentUser.id} is not member of mandate {mandateId}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Not member of mandate" ) - - if not membership.enabled: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Mandate membership is disabled" - ) - - ctx.mandateId = mandateId - - # Load roles via Junction Table - ctx.roleIds = rootInterface.getRoleIdsForUserMandate(membership.id) if featureInstanceId: - # Check feature access - ALSO for SysAdmin! + # Check feature access access = rootInterface.getFeatureAccess(currentUser.id, featureInstanceId) - if not access: + + if access: + # User has access - load their instance roles + if not access.enabled: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Feature access is disabled" + ) + ctx.featureInstanceId = featureInstanceId + instanceRoleIds = rootInterface.getRoleIdsForFeatureAccess(access.id) + ctx.roleIds.extend(instanceRoleIds) + elif isSysAdmin: + # SysAdmin can access any feature instance for admin operations + ctx.featureInstanceId = featureInstanceId + logger.debug(f"SysAdmin {currentUser.id} accessing feature instance {featureInstanceId} without explicit access") + else: + # Regular user without access - denied logger.warning(f"User {currentUser.id} has no access to feature instance {featureInstanceId}") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="No access to feature instance" ) - - if not access.enabled: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Feature access is disabled" - ) - - ctx.featureInstanceId = featureInstanceId - - # Add instance roles - instanceRoleIds = rootInterface.getRoleIdsForFeatureAccess(access.id) - ctx.roleIds.extend(instanceRoleIds) return ctx diff --git a/modules/features/aichat/mainAiChat.py b/modules/features/aichat/mainAiChat.py deleted file mode 100644 index 2e6514e6..00000000 --- a/modules/features/aichat/mainAiChat.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -AIChat Feature Container - Main Module. -Handles feature initialization and RBAC catalog registration. - -AIChat is the dynamic chat workflow feature that handles: -- AI-powered document processing -- Dynamic workflow execution -- Automation definitions -""" - -import logging -from typing import Dict, List, Any - -logger = logging.getLogger(__name__) - -# Feature metadata -FEATURE_CODE = "chatworkflow" -FEATURE_LABEL = {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"} -FEATURE_ICON = "mdi-message-cog" - -# UI Objects for RBAC catalog -UI_OBJECTS = [ - { - "objectKey": "ui.feature.aichat.workflows", - "label": {"en": "Workflows", "de": "Workflows", "fr": "Workflows"}, - "meta": {"area": "workflows"} - }, - { - "objectKey": "ui.feature.aichat.automations", - "label": {"en": "Automations", "de": "Automatisierungen", "fr": "Automatisations"}, - "meta": {"area": "automations"} - }, - { - "objectKey": "ui.feature.aichat.logs", - "label": {"en": "Logs", "de": "Logs", "fr": "Journaux"}, - "meta": {"area": "logs"} - }, -] - -# Resource Objects for RBAC catalog -RESOURCE_OBJECTS = [ - { - "objectKey": "resource.feature.aichat.workflow.start", - "label": {"en": "Start Workflow", "de": "Workflow starten", "fr": "Démarrer workflow"}, - "meta": {"endpoint": "/api/chat/playground/start", "method": "POST"} - }, - { - "objectKey": "resource.feature.aichat.workflow.stop", - "label": {"en": "Stop Workflow", "de": "Workflow stoppen", "fr": "Arrêter workflow"}, - "meta": {"endpoint": "/api/chat/playground/stop/{workflowId}", "method": "POST"} - }, - { - "objectKey": "resource.feature.aichat.workflow.delete", - "label": {"en": "Delete Workflow", "de": "Workflow löschen", "fr": "Supprimer workflow"}, - "meta": {"endpoint": "/api/chat/playground/workflow/{workflowId}", "method": "DELETE"} - }, -] - -# Template roles for this feature -TEMPLATE_ROLES = [ - { - "roleLabel": "workflow-admin", - "description": { - "en": "Workflow Administrator - Full access to workflow configuration and execution", - "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", - "fr": "Administrateur workflow - Accès complet à la configuration et exécution" - } - }, - { - "roleLabel": "workflow-editor", - "description": { - "en": "Workflow Editor - Create and modify workflows", - "de": "Workflow-Editor - Workflows erstellen und bearbeiten", - "fr": "Éditeur workflow - Créer et modifier les workflows" - } - }, - { - "roleLabel": "workflow-viewer", - "description": { - "en": "Workflow Viewer - View workflows and execution results", - "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", - "fr": "Visualiseur workflow - Consulter les workflows et résultats" - } - }, -] - - -def getFeatureDefinition() -> Dict[str, Any]: - """Return the feature definition for registration.""" - return { - "code": FEATURE_CODE, - "label": FEATURE_LABEL, - "icon": FEATURE_ICON - } - - -def getUiObjects() -> List[Dict[str, Any]]: - """Return UI objects for RBAC catalog registration.""" - return UI_OBJECTS - - -def getResourceObjects() -> List[Dict[str, Any]]: - """Return resource objects for RBAC catalog registration.""" - return RESOURCE_OBJECTS - - -def getTemplateRoles() -> List[Dict[str, Any]]: - """Return template roles for this feature.""" - return TEMPLATE_ROLES - - -def registerFeature(catalogService) -> bool: - """ - Register this feature's RBAC objects in the catalog. - - Args: - catalogService: The RBAC catalog service instance - - Returns: - True if registration was successful - """ - try: - # Register UI objects - for uiObj in UI_OBJECTS: - catalogService.registerUiObject( - featureCode=FEATURE_CODE, - objectKey=uiObj["objectKey"], - label=uiObj["label"], - meta=uiObj.get("meta") - ) - - # Register Resource objects - for resObj in RESOURCE_OBJECTS: - catalogService.registerResourceObject( - featureCode=FEATURE_CODE, - objectKey=resObj["objectKey"], - label=resObj["label"], - meta=resObj.get("meta") - ) - - logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") - return True - - except Exception as e: - logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") - return False - - -async def onStart(eventUser) -> None: - """ - Called when the feature container starts. - Initializes AI connectors for model registry. - """ - try: - from modules.aicore.aicoreModelRegistry import modelRegistry - modelRegistry.ensureConnectorsRegistered() - logger.info(f"Feature '{FEATURE_CODE}' started - AI connectors initialized") - except Exception as e: - logger.error(f"Feature '{FEATURE_CODE}' failed to initialize AI connectors: {e}") - - -async def onStop(eventUser) -> None: - """Called when the feature container stops.""" - logger.info(f"Feature '{FEATURE_CODE}' stopped") diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py index a0b8ba0f..88828442 100644 --- a/modules/features/automation/mainAutomation.py +++ b/modules/features/automation/mainAutomation.py @@ -66,7 +66,13 @@ TEMPLATE_ROLES = [ "en": "Automation Administrator - Full access to automation configuration and execution", "de": "Automatisierungs-Administrator - Vollzugriff auf Automatisierungs-Konfiguration und Ausführung", "fr": "Administrateur automatisation - Accès complet à la configuration et exécution" - } + }, + "accessRules": [ + # Full UI access + {"context": "UI", "item": None, "view": True}, + # Full DATA access + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + ] }, { "roleLabel": "automation-editor", @@ -74,7 +80,15 @@ TEMPLATE_ROLES = [ "en": "Automation Editor - Create and modify automations", "de": "Automatisierungs-Editor - Automatisierungen erstellen und bearbeiten", "fr": "Éditeur automatisation - Créer et modifier les automatisations" - } + }, + "accessRules": [ + # UI access to definitions and templates - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.automation.definitions", "view": True}, + {"context": "UI", "item": "ui.feature.automation.templates", "view": True}, + {"context": "UI", "item": "ui.feature.automation.logs", "view": True}, + # Group-level DATA access + {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "n"}, + ] }, { "roleLabel": "automation-viewer", @@ -82,7 +96,14 @@ TEMPLATE_ROLES = [ "en": "Automation Viewer - View automations and execution results", "de": "Automatisierungs-Betrachter - Automatisierungen und Ausführungsergebnisse einsehen", "fr": "Visualiseur automatisation - Consulter les automatisations et résultats" - } + }, + "accessRules": [ + # UI access to view only - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.automation.definitions", "view": True}, + {"context": "UI", "item": "ui.feature.automation.logs", "view": True}, + # Read-only DATA access (my level) + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ] }, ] diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 7db04c46..160addca 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -367,7 +367,9 @@ class ChatObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if operation == "create": diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 560a77b9..451a28f8 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -51,7 +51,15 @@ TEMPLATE_ROLES = [ "en": "Chatbot Administrator - Full access to chatbot settings and all conversations", "de": "Chatbot-Administrator - Vollzugriff auf Chatbot-Einstellungen und alle Konversationen", "fr": "Administrateur chatbot - Accès complet aux paramètres et conversations" - } + }, + "accessRules": [ + # Full UI access + {"context": "UI", "item": None, "view": True}, + # Full DATA access + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + # Resource access + {"context": "RESOURCE", "item": "resource.feature.chatbot.start", "view": True}, + ] }, { "roleLabel": "chatbot-user", @@ -59,7 +67,15 @@ TEMPLATE_ROLES = [ "en": "Chatbot User - Use chatbot and view own conversations", "de": "Chatbot-Benutzer - Chatbot nutzen und eigene Konversationen einsehen", "fr": "Utilisateur chatbot - Utiliser le chatbot et consulter ses conversations" - } + }, + "accessRules": [ + # UI access to conversations - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.chatbot.conversations", "view": True}, + # Own DATA access (my level) + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, + # Resource access + {"context": "RESOURCE", "item": "resource.feature.chatbot.start", "view": True}, + ] }, ] diff --git a/modules/features/neutralizer/datamodelFeatureNeutralizer.py b/modules/features/neutralization/datamodelFeatureNeutralizer.py similarity index 100% rename from modules/features/neutralizer/datamodelFeatureNeutralizer.py rename to modules/features/neutralization/datamodelFeatureNeutralizer.py diff --git a/modules/features/neutralizer/interfaceFeatureNeutralizer.py b/modules/features/neutralization/interfaceFeatureNeutralizer.py similarity index 98% rename from modules/features/neutralizer/interfaceFeatureNeutralizer.py rename to modules/features/neutralization/interfaceFeatureNeutralizer.py index 47439d62..970f51ff 100644 --- a/modules/features/neutralizer/interfaceFeatureNeutralizer.py +++ b/modules/features/neutralization/interfaceFeatureNeutralizer.py @@ -8,7 +8,7 @@ Handles CRUD operations for neutralization configuration and attributes. import logging from typing import Dict, List, Any, Optional -from modules.features.neutralizer.datamodelFeatureNeutralizer import ( +from modules.features.neutralization.datamodelFeatureNeutralizer import ( DataNeutraliserConfig, DataNeutralizerAttributes, ) diff --git a/modules/features/neutralizer/mainNeutralizePlayground.py b/modules/features/neutralization/mainNeutralizePlayground.py similarity index 100% rename from modules/features/neutralizer/mainNeutralizePlayground.py rename to modules/features/neutralization/mainNeutralizePlayground.py diff --git a/modules/features/neutralizer/mainNeutralizer.py b/modules/features/neutralization/mainNeutralizer.py similarity index 76% rename from modules/features/neutralizer/mainNeutralizer.py rename to modules/features/neutralization/mainNeutralizer.py index 44d495c4..d05f2b3f 100644 --- a/modules/features/neutralizer/mainNeutralizer.py +++ b/modules/features/neutralization/mainNeutralizer.py @@ -18,17 +18,17 @@ FEATURE_ICON = "mdi-shield-check" # UI Objects for RBAC catalog UI_OBJECTS = [ { - "objectKey": "ui.feature.neutralizer.playground", + "objectKey": "ui.feature.neutralization.playground", "label": {"en": "Playground", "de": "Spielwiese", "fr": "Bac à sable"}, "meta": {"area": "playground"} }, { - "objectKey": "ui.feature.neutralizer.config", + "objectKey": "ui.feature.neutralization.config", "label": {"en": "Configuration", "de": "Konfiguration", "fr": "Configuration"}, "meta": {"area": "config"} }, { - "objectKey": "ui.feature.neutralizer.attributes", + "objectKey": "ui.feature.neutralization.attributes", "label": {"en": "Attributes", "de": "Attribute", "fr": "Attributs"}, "meta": {"area": "attributes"} }, @@ -37,17 +37,17 @@ UI_OBJECTS = [ # Resource Objects for RBAC catalog RESOURCE_OBJECTS = [ { - "objectKey": "resource.feature.neutralizer.process.text", + "objectKey": "resource.feature.neutralization.process.text", "label": {"en": "Process Text", "de": "Text verarbeiten", "fr": "Traiter texte"}, "meta": {"endpoint": "/api/neutralization/process/text", "method": "POST"} }, { - "objectKey": "resource.feature.neutralizer.process.files", + "objectKey": "resource.feature.neutralization.process.files", "label": {"en": "Process Files", "de": "Dateien verarbeiten", "fr": "Traiter fichiers"}, "meta": {"endpoint": "/api/neutralization/process/files", "method": "POST"} }, { - "objectKey": "resource.feature.neutralizer.config.update", + "objectKey": "resource.feature.neutralization.config.update", "label": {"en": "Update Config", "de": "Konfiguration aktualisieren", "fr": "Mettre à jour config"}, "meta": {"endpoint": "/api/neutralization/config", "method": "PUT"} }, @@ -61,7 +61,13 @@ TEMPLATE_ROLES = [ "en": "Neutralization Administrator - Full access to neutralization settings and data", "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" - } + }, + "accessRules": [ + # Full UI access (all views including admin views) + {"context": "UI", "item": None, "view": True}, + # Full DATA access + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + ] }, { "roleLabel": "neutralization-analyst", @@ -69,7 +75,14 @@ TEMPLATE_ROLES = [ "en": "Neutralization Analyst - Analyze and process neutralization data", "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" - } + }, + "accessRules": [ + # UI access to specific views - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.neutralization.playground", "view": True}, + {"context": "UI", "item": "ui.feature.neutralization.attributes", "view": True}, + # Group-level DATA access (read-only for sensitive config) + {"context": "DATA", "item": None, "view": True, "read": "g", "create": "n", "update": "n", "delete": "n"}, + ] }, ] diff --git a/modules/features/neutralizer/routeFeatureNeutralizer.py b/modules/features/neutralization/routeFeatureNeutralizer.py similarity index 100% rename from modules/features/neutralizer/routeFeatureNeutralizer.py rename to modules/features/neutralization/routeFeatureNeutralizer.py diff --git a/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py similarity index 98% rename from modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py rename to modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py index f9e65284..4351f400 100644 --- a/modules/features/neutralizer/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py @@ -13,8 +13,8 @@ import re import json from typing import Dict, List, Any, Optional -from modules.features.neutralizer.datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes -from modules.features.neutralizer.interfaceFeatureNeutralizer import InterfaceFeatureNeutralizer +from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes +from modules.features.neutralization.interfaceFeatureNeutralizer import InterfaceFeatureNeutralizer # Import all necessary classes and functions for neutralization from .subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute diff --git a/modules/features/neutralizer/serviceNeutralization/subParseString.py b/modules/features/neutralization/serviceNeutralization/subParseString.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subParseString.py rename to modules/features/neutralization/serviceNeutralization/subParseString.py diff --git a/modules/features/neutralizer/serviceNeutralization/subPatterns.py b/modules/features/neutralization/serviceNeutralization/subPatterns.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subPatterns.py rename to modules/features/neutralization/serviceNeutralization/subPatterns.py diff --git a/modules/features/neutralizer/serviceNeutralization/subProcessBinary.py b/modules/features/neutralization/serviceNeutralization/subProcessBinary.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subProcessBinary.py rename to modules/features/neutralization/serviceNeutralization/subProcessBinary.py diff --git a/modules/features/neutralizer/serviceNeutralization/subProcessCommon.py b/modules/features/neutralization/serviceNeutralization/subProcessCommon.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subProcessCommon.py rename to modules/features/neutralization/serviceNeutralization/subProcessCommon.py diff --git a/modules/features/neutralizer/serviceNeutralization/subProcessList.py b/modules/features/neutralization/serviceNeutralization/subProcessList.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subProcessList.py rename to modules/features/neutralization/serviceNeutralization/subProcessList.py diff --git a/modules/features/neutralizer/serviceNeutralization/subProcessText.py b/modules/features/neutralization/serviceNeutralization/subProcessText.py similarity index 100% rename from modules/features/neutralizer/serviceNeutralization/subProcessText.py rename to modules/features/neutralization/serviceNeutralization/subProcessText.py diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 5a8c6d70..7a96afaa 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -742,7 +742,9 @@ class RealEstateObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if operation == "create": diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 76f658ba..d8447b96 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -41,7 +41,8 @@ RESOURCE_OBJECTS = [ }, ] -# Template roles for this feature +# Template roles for this feature with AccessRules +# IMPORTANT: item uses vollqualifizierte ObjectKeys (gemäss Navigation-API-Konzept) TEMPLATE_ROLES = [ { "roleLabel": "realestate-admin", @@ -49,7 +50,16 @@ TEMPLATE_ROLES = [ "en": "Real Estate Administrator - Full access to all property data and settings", "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", "fr": "Administrateur immobilier - Accès complet aux données et paramètres" - } + }, + "accessRules": [ + # Full UI access (all views including admin views) + {"context": "UI", "item": None, "view": True}, + # Full DATA access + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + # Admin resources + {"context": "RESOURCE", "item": "resource.feature.realestate.project.create", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.realestate.project.delete", "view": True}, + ] }, { "roleLabel": "realestate-manager", @@ -57,7 +67,16 @@ TEMPLATE_ROLES = [ "en": "Real Estate Manager - Manage properties and tenants", "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" - } + }, + "accessRules": [ + # UI access to main views - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.realestate.projects", "view": True}, + {"context": "UI", "item": "ui.feature.realestate.parcels", "view": True}, + # Group-level DATA access + {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, + # Resource: create projects + {"context": "RESOURCE", "item": "resource.feature.realestate.project.create", "view": True}, + ] }, { "roleLabel": "realestate-viewer", @@ -65,7 +84,14 @@ TEMPLATE_ROLES = [ "en": "Real Estate Viewer - View property information", "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", "fr": "Visualiseur immobilier - Consulter les informations immobilières" - } + }, + "accessRules": [ + # UI access to view-only views - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.realestate.projects", "view": True}, + {"context": "UI", "item": "ui.feature.realestate.parcels", "view": True}, + # Read-only DATA access (my records) + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ] }, ] diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index 7fc90bdb..729793b4 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -171,7 +171,9 @@ class TrusteeObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if not permissions.view: @@ -196,7 +198,9 @@ class TrusteeObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if not permissions.view: @@ -258,7 +262,9 @@ class TrusteeObjects: modelClass=TrusteeOrganisation, currentUser=self.currentUser, recordFilter=None, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) logger.debug(f"getAllOrganisations: getRecordsetWithRBAC returned {len(records)} records") @@ -349,7 +355,9 @@ class TrusteeObjects: modelClass=TrusteeRole, currentUser=self.currentUser, recordFilter=None, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all roles @@ -457,7 +465,9 @@ class TrusteeObjects: modelClass=TrusteeAccess, currentUser=self.currentUser, recordFilter=None, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all records @@ -514,7 +524,9 @@ class TrusteeObjects: modelClass=TrusteeAccess, currentUser=self.currentUser, recordFilter={"organisationId": organisationId}, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -529,7 +541,9 @@ class TrusteeObjects: modelClass=TrusteeAccess, currentUser=self.currentUser, recordFilter={"userId": userId}, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all records @@ -644,7 +658,9 @@ class TrusteeObjects: modelClass=TrusteeContract, currentUser=self.currentUser, recordFilter=None, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -678,7 +694,9 @@ class TrusteeObjects: modelClass=TrusteeContract, currentUser=self.currentUser, recordFilter={"organisationId": organisationId}, - orderBy="label" + orderBy="label", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -785,7 +803,9 @@ class TrusteeObjects: modelClass=TrusteeDocument, currentUser=self.currentUser, recordFilter=None, - orderBy="documentName" + orderBy="documentName", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -826,7 +846,9 @@ class TrusteeObjects: modelClass=TrusteeDocument, currentUser=self.currentUser, recordFilter={"contractId": contractId}, - orderBy="documentName" + orderBy="documentName", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -934,7 +956,9 @@ class TrusteeObjects: modelClass=TrusteePosition, currentUser=self.currentUser, recordFilter=None, - orderBy="valuta" + orderBy="valuta", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -975,7 +999,9 @@ class TrusteeObjects: modelClass=TrusteePosition, currentUser=self.currentUser, recordFilter={"contractId": contractId}, - orderBy="valuta" + orderBy="valuta", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -990,7 +1016,9 @@ class TrusteeObjects: modelClass=TrusteePosition, currentUser=self.currentUser, recordFilter={"organisationId": organisationId}, - orderBy="valuta" + orderBy="valuta", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -1078,15 +1106,46 @@ class TrusteeObjects: return None return TrusteePositionDocument(**{k: v for k, v in records[0].items() if not k.startswith("_")}) + def updatePositionDocument(self, linkId: str, data: Dict[str, Any]) -> Optional[TrusteePositionDocument]: + """Update a position-document link.""" + # Check permission + if not self.checkCombinedPermission(TrusteePositionDocument, "update"): + logger.warning(f"User {self.userId} lacks permission to update position-document link") + return None + + # Verify link exists and belongs to this instance + existing = self.db.getRecordset(TrusteePositionDocument, recordFilter={"id": linkId}) + if not existing: + logger.warning(f"Position-document link {linkId} not found") + return None + + existingRecord = existing[0] + if existingRecord.get("featureInstanceId") != self.featureInstanceId: + logger.warning(f"Link {linkId} belongs to different instance") + return None + + # Prevent changing context fields + data.pop("id", None) + data.pop("mandateId", None) + data.pop("featureInstanceId", None) + + updatedRecord = self.db.recordModify(TrusteePositionDocument, linkId, data) + if updatedRecord and updatedRecord.get("id"): + return TrusteePositionDocument(**{k: v for k, v in updatedRecord.items() if not k.startswith("_")}) + return None + def getAllPositionDocuments(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all position-document links with RBAC filtering + feature-level access filtering.""" - # Step 1: System RBAC filtering + # Step 1: System RBAC filtering with per-row permissions records = getRecordsetWithRBAC( connector=self.db, modelClass=TrusteePositionDocument, currentUser=self.currentUser, recordFilter=None, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + enrichPermissions=True ) # Step 2: Feature-level filtering based on trustee.access @@ -1121,7 +1180,9 @@ class TrusteeObjects: modelClass=TrusteePositionDocument, currentUser=self.currentUser, recordFilter={"positionId": positionId}, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access @@ -1136,7 +1197,9 @@ class TrusteeObjects: modelClass=TrusteePositionDocument, currentUser=self.currentUser, recordFilter={"documentId": documentId}, - orderBy="id" + orderBy="id", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # Step 2: Feature-level filtering based on trustee.access diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 5ee0ed08..311293f6 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -45,6 +45,31 @@ UI_OBJECTS = [ }, ] +# DATA Objects for RBAC catalog (tables/entities) +# Used for AccessRules on data-level permissions +DATA_OBJECTS = [ + { + "objectKey": "data.feature.trustee.TrusteePosition", + "label": {"en": "Position", "de": "Position", "fr": "Position"}, + "meta": {"table": "TrusteePosition", "fields": ["id", "label", "description", "organisationId"]} + }, + { + "objectKey": "data.feature.trustee.TrusteeDocument", + "label": {"en": "Document", "de": "Dokument", "fr": "Document"}, + "meta": {"table": "TrusteeDocument", "fields": ["id", "filename", "mimeType", "fileSize", "uploadDate"]} + }, + { + "objectKey": "data.feature.trustee.TrusteePositionDocument", + "label": {"en": "Position-Document Assignment", "de": "Position-Dokument Zuordnung", "fr": "Assignation Position-Document"}, + "meta": {"table": "TrusteePositionDocument", "fields": ["id", "positionId", "documentId"]} + }, + { + "objectKey": "data.feature.trustee.*", + "label": {"en": "All Trustee Data", "de": "Alle Treuhand-Daten", "fr": "Toutes les données fiduciaires"}, + "meta": {"wildcard": True, "description": "Wildcard for all trustee data tables"} + }, +] + # Resource Objects for RBAC catalog # Note: organisations and contracts removed - feature instance = organisation RESOURCE_OBJECTS = [ @@ -88,6 +113,7 @@ RESOURCE_OBJECTS = [ # Template roles for this feature with AccessRules # Each role defines default UI and DATA permissions # Note: UI item=None means ALL views, specific items restrict to named views +# IMPORTANT: item uses vollqualifizierte ObjectKeys (gemäss Navigation-API-Konzept) TEMPLATE_ROLES = [ { "roleLabel": "trustee-admin", @@ -102,7 +128,7 @@ TEMPLATE_ROLES = [ # Full DATA access {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, # Admin resource: manage instance roles - {"context": "RESOURCE", "item": "instance-roles.manage", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.trustee.instance-roles.manage", "view": True}, ] }, { @@ -113,11 +139,11 @@ TEMPLATE_ROLES = [ "fr": "Comptable fiduciaire - Gérer les données comptables et financières" }, "accessRules": [ - # UI access to main views (not admin views) - {"context": "UI", "item": "dashboard", "view": True}, - {"context": "UI", "item": "positions", "view": True}, - {"context": "UI", "item": "documents", "view": True}, - {"context": "UI", "item": "position-documents", "view": True}, + # UI access to main views (not admin views) - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.position-documents", "view": True}, # Group-level DATA access {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, ] @@ -130,12 +156,15 @@ TEMPLATE_ROLES = [ "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" }, "accessRules": [ - # UI access to main views only (read-only focus) - {"context": "UI", "item": "dashboard", "view": True}, - {"context": "UI", "item": "positions", "view": True}, - {"context": "UI", "item": "documents", "view": True}, - # Own records only (MY level) - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + # UI access to main views only (read-only focus) - vollqualifizierte ObjectKeys + {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.position-documents", "view": True}, + # Own records only (MY level) - explizite Regeln pro Tabelle + {"context": "DATA", "item": "data.feature.trustee.TrusteePosition", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + {"context": "DATA", "item": "data.feature.trustee.TrusteeDocument", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + {"context": "DATA", "item": "data.feature.trustee.TrusteePositionDocument", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, ] }, ] @@ -165,6 +194,11 @@ def getTemplateRoles() -> List[Dict[str, Any]]: return TEMPLATE_ROLES +def getDataObjects() -> List[Dict[str, Any]]: + """Return DATA objects for RBAC catalog registration.""" + return DATA_OBJECTS + + def registerFeature(catalogService) -> bool: """ Register this feature's RBAC objects in the catalog. @@ -194,10 +228,19 @@ def registerFeature(catalogService) -> bool: meta=resObj.get("meta") ) + # Register DATA objects (tables/entities) + for dataObj in DATA_OBJECTS: + catalogService.registerDataObject( + featureCode=FEATURE_CODE, + objectKey=dataObj["objectKey"], + label=dataObj["label"], + meta=dataObj.get("meta") + ) + # Sync template roles to database (with AccessRules) _syncTemplateRolesToDb() - logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI, {len(RESOURCE_OBJECTS)} resource, {len(DATA_OBJECTS)} data objects") return True except Exception as e: diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index f2b2ead2..9b1b1fca 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -282,12 +282,13 @@ async def get_document_options( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: - """Get document options for select dropdowns. Returns: [{ value, label }]""" + """Get document options for select dropdowns. Returns: [{ id, value, label }]""" mandateId = await _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllDocuments(None) items = result.items if hasattr(result, 'items') else result - return [{"value": d.id, "label": d.documentName or d.id} for d in items] + # Include 'id' for FK resolution in tables + return [{"id": d.id, "value": d.id, "label": d.documentName or d.id} for d in items] @router.get("/{instanceId}/positions/options", response_model=List[Dict[str, Any]]) @@ -297,7 +298,7 @@ async def get_position_options( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: - """Get position options for select dropdowns. Returns: [{ value, label }]""" + """Get position options for select dropdowns. Returns: [{ id, value, label }]""" mandateId = await _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(None) @@ -313,7 +314,8 @@ async def get_position_options( parts.append(p.desc[:30]) return " - ".join(parts) if parts else p.id - return [{"value": p.id, "label": _makePositionLabel(p)} for p in items] + # Include 'id' for FK resolution in tables + return [{"id": p.id, "value": p.id, "label": _makePositionLabel(p)} for p in items] # ============================================================================ @@ -1166,15 +1168,18 @@ async def delete_position( # ===== Position-Document Link Routes ===== -@router.get("/{instanceId}/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents") @limiter.limit("30/minute") async def get_position_documents( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) -) -> PaginatedResponse[TrusteePositionDocument]: - """Get all position-document links with optional pagination.""" +) -> Dict[str, Any]: + """Get all position-document links with optional pagination. + + Each item includes _permissions: { canUpdate, canDelete } for row-level permission UI. + """ mandateId = await _validateInstanceAccess(instanceId, context) paginationParams = _parsePagination(pagination) @@ -1182,18 +1187,18 @@ async def get_position_documents( result = interface.getAllPositionDocuments(paginationParams) if paginationParams: - return PaginatedResponse( - items=result.items, - pagination=PaginationMetadata( - currentPage=paginationParams.page or 1, - pageSize=paginationParams.pageSize or 20, - totalItems=result.totalItems, - totalPages=result.totalPages, - sort=paginationParams.sort if paginationParams else [], - filters=paginationParams.filters if paginationParams else None - ) - ) - return PaginatedResponse(items=result.items, pagination=None) + return { + "items": result.items, + "pagination": { + "currentPage": paginationParams.page or 1, + "pageSize": paginationParams.pageSize or 20, + "totalItems": result.totalItems, + "totalPages": result.totalPages, + "sort": paginationParams.sort if paginationParams else [], + "filters": paginationParams.filters if paginationParams else None + } + } + return {"items": result.items, "pagination": None} @router.get("/{instanceId}/position-documents/{linkId}", response_model=TrusteePositionDocument) @@ -1262,6 +1267,25 @@ async def create_position_document( return result +@router.put("/{instanceId}/position-documents/{linkId}", response_model=TrusteePositionDocument) +@limiter.limit("10/minute") +async def update_position_document( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + linkId: str = Path(...), + data: TrusteePositionDocument = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> TrusteePositionDocument: + """Update a position-document link.""" + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + result = interface.updatePositionDocument(linkId, data.model_dump(exclude_unset=True)) + if not result: + raise HTTPException(status_code=400, detail="Failed to update link") + return result + + @router.delete("/{instanceId}/position-documents/{linkId}") @limiter.limit("10/minute") async def delete_position_document( @@ -1505,10 +1529,10 @@ async def update_instance_role_rule( updateData["delete"] = ruleData["delete"] if not updateData: - return existingRule + return existingRules[0] try: - updated = rootInterface.db.recordUpdate(AccessRule, ruleId, updateData) + updated = rootInterface.db.recordModify(AccessRule, ruleId, updateData) return updated except Exception as e: logger.error(f"Error updating AccessRule: {e}") diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 4b291537..472e77f9 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -447,29 +447,33 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # Standard tables with typical access patterns + # System tables only - NOT feature-specific tables! + # Feature tables (TrusteeXXX, Projekt, etc.) are handled by FEATURE-TEMPLATE roles. # NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag - standardTables = [ - "UserConnection", "DataNeutraliserConfig", "DataNeutralizerAttributes", - "ChatWorkflow", "Prompt", "Projekt", "Parzelle", "Dokument", - "Gemeinde", "Kanton", "Land", - "TrusteeOrganisation", "TrusteeRole", "TrusteeAccess", "TrusteeContract", - "TrusteeDocument", "TrusteePosition", "TrusteePositionDocument" + # + # Proper format: Just table names for DATA context (item="TableName") + # The full data.system.TableName format is for catalog registration only. + + # FileItem and UserConnection: All users (user, admin, viewer) only MY-level CRUD + restrictedTables = [ + "UserConnection", # User connections/sessions - only own records + "FileItem", # Uploaded files - only own files ] - for table in standardTables: - # Admin gets full group-level access (highest role-based permission) + for table in restrictedTables: + # Admin: Only MY-level access (not group-level!) if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, item=table, view=True, - read=AccessLevel.GROUP, - create=AccessLevel.GROUP, - update=AccessLevel.GROUP, - delete=AccessLevel.GROUP, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, )) + # User: MY-level CRUD if userId: tableRules.append(AccessRule( roleId=userId, @@ -481,6 +485,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: update=AccessLevel.MY, delete=AccessLevel.MY, )) + # Viewer: MY-level read-only if viewerId: tableRules.append(AccessRule( roleId=viewerId, @@ -493,6 +498,80 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) + # Prompt: Special rule - CRUD for MY + Read for GROUP + # Each user can manage own prompts (m) but can read group prompts (g) + if adminId: + # Admin: MY-level CRUD + GROUP-level read + tableRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item="Prompt", + view=True, + read=AccessLevel.GROUP, # Can read group prompts + create=AccessLevel.MY, # Can create own prompts + update=AccessLevel.MY, # Can update own prompts + delete=AccessLevel.MY, # Can delete own prompts + )) + if userId: + # User: MY-level CRUD + GROUP-level read + tableRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item="Prompt", + view=True, + read=AccessLevel.GROUP, # Can read group prompts + create=AccessLevel.MY, # Can create own prompts + update=AccessLevel.MY, # Can update own prompts + delete=AccessLevel.MY, # Can delete own prompts + )) + if viewerId: + # Viewer: MY-level read + GROUP-level read + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="Prompt", + view=True, + read=AccessLevel.GROUP, # Can read group prompts + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # Invitation: Standard group-level access + if adminId: + tableRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item="Invitation", + view=True, + read=AccessLevel.GROUP, + create=AccessLevel.GROUP, + update=AccessLevel.GROUP, + delete=AccessLevel.GROUP, + )) + if userId: + tableRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item="Invitation", + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="Invitation", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + # AuthEvent table - Audit logs (no delete allowed for audit integrity!) # SysAdmin can delete via isSysAdmin bypass, but regular admins cannot if adminId: @@ -541,27 +620,51 @@ def _createUiContextRules(db: DatabaseConnector) -> None: Create UI context rules for controlling UI element visibility. Uses roleId instead of roleLabel. + Creates rules for system pages based on NAVIGATION_SECTIONS. + Admin pages require admin role, public pages are available to all. + NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag. Args: db: Database connector instance """ - uiRules = [] + from modules.system.mainSystem import NAVIGATION_SECTIONS - # All roles get full UI access by default (no sysadmin - that's a flag) - for roleLabel in ["admin", "user", "viewer"]: - roleId = _getRoleId(db, roleLabel) - if roleId: - uiRules.append(AccessRule( - roleId=roleId, - context=AccessRuleContext.UI, - item=None, - view=True, - read=None, - create=None, - update=None, - delete=None, - )) + uiRules = [] + adminId = _getRoleId(db, "admin") + userId = _getRoleId(db, "user") + viewerId = _getRoleId(db, "viewer") + + # Create rules based on navigation sections + for section in NAVIGATION_SECTIONS: + isAdminSection = section.get("adminOnly", False) + + for item in section.get("items", []): + objectKey = item.get("objectKey") + isPublic = item.get("public", False) + isAdminOnly = item.get("adminOnly", False) or isAdminSection + + if isAdminOnly: + # Admin-only pages: only admin role + if adminId: + uiRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.UI, + item=objectKey, + view=True, + read=None, create=None, update=None, delete=None, + )) + else: + # Public/normal pages: all roles + for roleId in [adminId, userId, viewerId]: + if roleId: + uiRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.UI, + item=objectKey, + view=True, + read=None, create=None, update=None, delete=None, + )) for rule in uiRules: db.recordCreate(AccessRule, rule) diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 9c769e7c..a7dfc689 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -227,7 +227,8 @@ class AppObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId ) if operation == "create": diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index 25b3d3d6..3c4d35ad 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -367,7 +367,9 @@ class ChatObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if operation == "create": diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index a26e8c98..10c47a19 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -319,7 +319,9 @@ class ComponentObjects: permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName + tableName, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if operation == "create": diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 8ceefa6a..b34b2e36 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -30,6 +30,7 @@ def getRecordsetWithRBAC( limit: int = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None, + enrichPermissions: bool = False, ) -> List[Dict[str, Any]]: """ Get records with RBAC filtering applied at database level. @@ -47,9 +48,11 @@ def getRecordsetWithRBAC( limit: Maximum number of records to return mandateId: Explicit mandate context (from request header). Required for GROUP access. featureInstanceId: Explicit feature instance context + enrichPermissions: If True, adds _permissions field to each record with row-level + permissions { canUpdate, canDelete } based on RBAC rules and _createdBy Returns: - List of filtered records + List of filtered records (with _permissions if enrichPermissions=True) """ table = modelClass.__name__ @@ -64,7 +67,12 @@ def getRecordsetWithRBAC( if isSysAdmin: # Direct access without RBAC filtering # Note: getRecordset doesn't support orderBy/limit - these are only used in RBAC path - return connector.getRecordset(modelClass, recordFilter=recordFilter) + records = connector.getRecordset(modelClass, recordFilter=recordFilter) + if enrichPermissions: + # SysAdmin has full permissions on all records + for record in records: + record["_permissions"] = {"canUpdate": True, "canDelete": True} + return records # Get RBAC permissions for this table # AccessRule table is always in DbApp database @@ -173,6 +181,12 @@ def getRecordsetWithRBAC( f"Could not parse JSONB field {fieldName}, keeping as string: {record[fieldName]}" ) + # Enrich records with row-level permissions if requested + if enrichPermissions: + records = _enrichRecordsWithPermissions( + records, permissions, currentUser + ) + return records except Exception as e: logger.error(f"Error loading records with RBAC from table {table}: {e}") @@ -292,3 +306,87 @@ def buildRbacWhereClause( } return None + + +def _enrichRecordsWithPermissions( + records: List[Dict[str, Any]], + permissions: UserPermissions, + currentUser: User +) -> List[Dict[str, Any]]: + """ + Enrich records with per-row permissions (_permissions field). + + The _permissions field contains: + - canUpdate: bool - whether current user can update this record + - canDelete: bool - whether current user can delete this record + + Logic: + - AccessLevel.ALL ('a'): User can update/delete all records + - AccessLevel.MY ('m'): User can only update/delete records where _createdBy == userId + - AccessLevel.GROUP ('g'): Same as MY for now (group-level ownership) + - AccessLevel.NONE ('n'): User cannot update/delete any records + + Args: + records: List of record dicts + permissions: UserPermissions with update/delete levels + currentUser: Current user object + + Returns: + Records with _permissions field added + """ + enriched = [] + userId = currentUser.id if currentUser else None + + for record in records: + recordCopy = dict(record) + createdBy = record.get("_createdBy") + + # Determine canUpdate + canUpdate = _checkRowPermission(permissions.update, userId, createdBy) + # Determine canDelete + canDelete = _checkRowPermission(permissions.delete, userId, createdBy) + + recordCopy["_permissions"] = { + "canUpdate": canUpdate, + "canDelete": canDelete + } + enriched.append(recordCopy) + + return enriched + + +def _checkRowPermission( + accessLevel: Optional[AccessLevel], + userId: Optional[str], + recordCreatedBy: Optional[str] +) -> bool: + """ + Check if user has permission for a specific row based on access level. + + Args: + accessLevel: The permission level (ALL, MY, GROUP, NONE) + userId: Current user's ID + recordCreatedBy: The _createdBy value of the record + + Returns: + True if user has permission, False otherwise + """ + if not accessLevel or accessLevel == AccessLevel.NONE: + return False + + if accessLevel == AccessLevel.ALL: + return True + + # MY and GROUP: Check ownership via _createdBy + if accessLevel in (AccessLevel.MY, AccessLevel.GROUP): + # If record has no _createdBy, allow access (can't verify ownership) + if not recordCreatedBy: + return True + # If no userId, can't verify - deny + if not userId: + return False + # Check ownership + return recordCreatedBy == userId + + # Unknown level - deny by default + return False diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 2c8408e3..1bb6be16 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -325,19 +325,16 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict current["delete"] = _mergeAccessLevel(current["delete"], rule.get("delete") or "n") # Handle UI context (views) + # Views are stored with full objectKey (e.g., ui.feature.trustee.dashboard) elif context == "UI" or context == AccessRuleContext.UI: ruleView = rule.get("view", False) if item: - # Specific view rule + # Store with full objectKey as per Navigation-API-Konzept permissions["views"][item] = permissions["views"].get(item, False) or ruleView elif ruleView: # item=None means all views - set a wildcard flag permissions["views"]["_all"] = True - # Derive view permissions from table permissions - # This allows UI navigation to be controlled by data access rights - _deriveViewPermissions(permissions) - return permissions except Exception as e: @@ -345,51 +342,6 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict return permissions -def _deriveViewPermissions(permissions: Dict[str, Any]) -> None: - """ - Derive UI view permissions from table/data permissions. - - Mapping: - - trustee-dashboard: always visible (basic access) - - trustee-positions: visible if READ on TrusteePosition - - trustee-documents: visible if READ on TrusteeDocument - - trustee-position-documents: visible if READ on TrusteePositionDocument - - trustee-instance-roles: visible only for admin roles - - This function modifies permissions["views"] in place. - """ - tables = permissions.get("tables", {}) - views = permissions.get("views", {}) - isAdmin = permissions.get("isAdmin", False) - - # If user has _all views permission, skip derivation - if views.get("_all"): - return - - # Dashboard is always visible for users with any access - if "trustee-dashboard" not in views: - views["trustee-dashboard"] = True - - # Positions view: requires READ on TrusteePosition - if "trustee-positions" not in views: - positionPerms = tables.get("TrusteePosition", {}) - views["trustee-positions"] = positionPerms.get("read", "n") != "n" - - # Documents view: requires READ on TrusteeDocument - if "trustee-documents" not in views: - documentPerms = tables.get("TrusteeDocument", {}) - views["trustee-documents"] = documentPerms.get("read", "n") != "n" - - # Position-Documents view: requires READ on TrusteePositionDocument - if "trustee-position-documents" not in views: - linkPerms = tables.get("TrusteePositionDocument", {}) - views["trustee-position-documents"] = linkPerms.get("read", "n") != "n" - - # Instance-roles (admin) view: requires admin role - if "trustee-instance-roles" not in views: - views["trustee-instance-roles"] = isAdmin - - def _mergeAccessLevel(current: str, new: str) -> str: """Merge two access levels, returning the highest.""" levels = {"n": 0, "m": 1, "g": 2, "a": 3} diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index a1990543..d125bc2c 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -19,6 +19,7 @@ import math from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role +from modules.datamodels.datamodelMembership import UserMandate from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.interfaces.interfaceDbApp import getInterface, getRootInterface @@ -77,11 +78,13 @@ async def get_permissions( ) # MULTI-TENANT: Get permissions using context (mandateId/featureInstanceId) - # For now, pass user - RBAC will be extended to use context in later phases + # Pass mandateId and featureInstanceId to load Feature-Instance roles permissions = interface.rbac.getUserPermissions( reqContext.user, accessContext, - item or "" + item or "", + mandateId=reqContext.mandateId, + featureInstanceId=reqContext.featureInstanceId ) return permissions @@ -166,32 +169,92 @@ async def get_all_permissions( result: Dict[str, Any] = {} - # MULTI-TENANT: Get role IDs from context (computed from mandateId/featureInstanceId) - roleIds = reqContext.roleIds or [] + # For UI/RESOURCE permissions: These are GLOBAL (not mandate-specific) + # System roles (admin, user, viewer) have global UI rules that apply without mandate context + + rootInterface = getRootInterface() + + # Start with roleIds from current mandate context (if any) + roleIds = list(reqContext.roleIds or []) + + # For UI/RESOURCE: Load system roles the user has across ALL their mandates + # This allows users to access system UI elements without needing a specific mandate header + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"userId": str(reqContext.user.id), "enabled": True} + ) + + # Collect all role IDs the user has across all mandates + for userMandate in userMandates: + mandateRoleIds = rootInterface.getRoleIdsForUserMandate(userMandate.get("id")) + for rid in mandateRoleIds: + if rid not in roleIds: + roleIds.append(rid) + + logger.debug(f"UI/RESOURCE permissions: User has {len(roleIds)} roles across all mandates") + if not roleIds and not reqContext.isSysAdmin: - # User has no roles, return empty permissions + # No roles at all, return empty permissions for ctx in contextsToFetch: result[ctx.value.lower()] = {} return result # Get all access rules for user's roles and requested contexts + # IMPORTANT: Use direct DB access without RBAC filtering! + # Otherwise we have a chicken-and-egg problem: need AccessRule read permission to calculate permissions allRules: Dict[AccessRuleContext, List[AccessRule]] = {} for ctx in contextsToFetch: allRules[ctx] = [] - # Get all rules for user's roles in this context + # Get all rules for user's roles - bypass RBAC filtering for roleId in roleIds: - rules = interface.getAccessRules( - roleId=str(roleId), - context=ctx, - pagination=None + ruleRecords = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": str(roleId), "context": ctx.value} ) - allRules[ctx].extend(rules) + for ruleRecord in ruleRecords: + # Convert dict to AccessRule object + cleanedRule = {k: v for k, v in ruleRecord.items() if not k.startswith("_")} + allRules[ctx].append(AccessRule(**cleanedRule)) # Build result: for each context, collect all unique items and calculate permissions for ctx in contextsToFetch: result[ctx.value.lower()] = {} - # Collect all unique items from rules + # Check for global rule (item=None) first - this grants access to ALL UI/RESOURCE items + # Calculate permissions directly from loaded rules (don't call getUserPermissions - it requires mandateId) + hasGlobalRule = False + globalView = False + globalRead = None + globalCreate = None + globalUpdate = None + globalDelete = None + + for rule in allRules[ctx]: + if rule.item is None: + hasGlobalRule = True + if rule.view: + globalView = True + if rule.read: + globalRead = rule.read.value if hasattr(rule.read, 'value') else rule.read + if rule.create: + globalCreate = rule.create.value if hasattr(rule.create, 'value') else rule.create + if rule.update: + globalUpdate = rule.update.value if hasattr(rule.update, 'value') else rule.update + if rule.delete: + globalDelete = rule.delete.value if hasattr(rule.delete, 'value') else rule.delete + + # If there's a global rule with view permission, add "_global" key + if hasGlobalRule and globalView: + logger.debug(f"Adding _global key for context {ctx.value} with view={globalView}") + result[ctx.value.lower()]["_global"] = { + "view": globalView, + "read": globalRead, + "create": globalCreate, + "update": globalUpdate, + "delete": globalDelete + } + + # Collect all unique items from rules (specific rules) items = set() for rule in allRules[ctx]: if rule.item: @@ -199,7 +262,11 @@ async def get_all_permissions( # For each item, calculate user permissions for item in sorted(items): - permissions = interface.rbac.getUserPermissions(reqContext.user, ctx, item) + permissions = interface.rbac.getUserPermissions( + reqContext.user, ctx, item, + mandateId=reqContext.mandateId, + featureInstanceId=reqContext.featureInstanceId + ) # Only include if user has view permission if permissions.view: result[ctx.value.lower()][item] = { @@ -1007,3 +1074,129 @@ async def delete_role( status_code=500, detail=f"Failed to delete role: {str(e)}" ) + + +# ============================================================================ +# RBAC Catalog Endpoints +# ============================================================================ + +@router.get("/catalog/objects", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getCatalogObjects( + request: Request, + context: Optional[str] = Query(None, description="Filter by context (DATA, UI, RESOURCE)"), + featureCode: Optional[str] = Query(None, description="Filter by feature code"), + mandateId: Optional[str] = Query(None, description="Filter by mandate's active features"), + reqContext: RequestContext = Depends(getRequestContext) # Available to all authenticated users +) -> Dict[str, Any]: + """ + Get available RBAC catalog objects. + Returns all registered DATA, UI and RESOURCE objects that can be used in AccessRules. + + Query Parameters: + - context: Optional filter by context type (DATA, UI, RESOURCE) + - featureCode: Optional filter by feature (e.g., "trustee") + - mandateId: Optional filter to only include objects from features active in this mandate + + Returns: + - Dictionary with objects grouped by context type, each with: + - objectKey: Dot-notation identifier (e.g., "data.feature.trustee.TrusteeContract") + - label: Multilingual label + - featureCode: Owning feature + - meta: Additional metadata (table name, fields, etc.) + + Examples: + - GET /api/rbac/catalog/objects → all objects + - GET /api/rbac/catalog/objects?context=DATA → only DATA objects + - GET /api/rbac/catalog/objects?featureCode=trustee → only trustee objects + - GET /api/rbac/catalog/objects?mandateId=xxx → objects from mandate's active features + """ + try: + from modules.security.rbacCatalog import getCatalogService + + catalog = getCatalogService() + + # If mandateId is provided, get active features for that mandate + activeFeatures = None + if mandateId: + try: + interface = getRootInterface() + # Get all feature instances for this mandate + from modules.datamodels.datamodelFeatures import FeatureInstance + instances = interface.db.getRecordset( + FeatureInstance, + recordFilter={"mandateId": mandateId, "enabled": True} + ) + activeFeatures = set(inst.get("featureCode") for inst in instances) + # Always include "system" feature + activeFeatures.add("system") + except Exception as e: + logger.warning(f"Could not get active features for mandate {mandateId}: {e}") + + if context: + # Single context filter + try: + accessContext = AccessRuleContext(context.upper()) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Invalid context '{context}'. Must be one of: DATA, UI, RESOURCE" + ) + + if accessContext == AccessRuleContext.DATA: + objects = catalog.getDataObjects(featureCode) + elif accessContext == AccessRuleContext.UI: + objects = catalog.getUiObjects(featureCode) + else: + objects = catalog.getResourceObjects(featureCode) + + # Filter by active features if mandateId was provided + if activeFeatures: + objects = [obj for obj in objects if obj.get("featureCode") in activeFeatures] + + return {context.upper(): objects} + else: + # All contexts + result = catalog.getAllCatalogObjects(featureCode) + + # Filter by active features if mandateId was provided + if activeFeatures: + for ctxKey in result: + result[ctxKey] = [obj for obj in result[ctxKey] if obj.get("featureCode") in activeFeatures] + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting catalog objects: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to get catalog objects: {str(e)}" + ) + + +@router.get("/catalog/stats", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getCatalogStats( + request: Request, + currentUser: User = Depends(requireSysAdmin) +) -> Dict[str, Any]: + """ + Get statistics about the RBAC catalog. + + Returns: + - Statistics about registered features, objects, and roles + """ + try: + from modules.security.rbacCatalog import getCatalogService + + catalog = getCatalogService() + return catalog.getCatalogStats() + + except Exception as e: + logger.error(f"Error getting catalog stats: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to get catalog stats: {str(e)}" + ) diff --git a/modules/security/rbacCatalog.py b/modules/security/rbacCatalog.py index f52adf21..a913a095 100644 --- a/modules/security/rbacCatalog.py +++ b/modules/security/rbacCatalog.py @@ -37,6 +37,7 @@ class RbacCatalogService: self._uiObjects: Dict[str, Dict[str, Any]] = {} self._resourceObjects: Dict[str, Dict[str, Any]] = {} + self._dataObjects: Dict[str, Dict[str, Any]] = {} # DATA objects (tables/entities) self._featureDefinitions: Dict[str, Dict[str, Any]] = {} self._templateRoles: Dict[str, List[Dict[str, Any]]] = {} self._initialized = True @@ -60,6 +61,29 @@ class RbacCatalogService: logger.error(f"Failed to register RESOURCE object {objectKey}: {e}") return False + def registerDataObject(self, featureCode: str, objectKey: str, label: Dict[str, str], meta: Optional[Dict[str, Any]] = None) -> bool: + """ + Register a DATA object (table/entity) for a feature. + + Args: + featureCode: Feature code (e.g., "trustee", "system") + objectKey: Dot-notation key (e.g., "data.feature.trustee.TrusteeContract") + label: Multilingual label dict + meta: Optional metadata (e.g., table name, fields list) + """ + try: + self._dataObjects[objectKey] = { + "objectKey": objectKey, + "featureCode": featureCode, + "label": label, + "meta": meta or {}, + "type": "DATA" + } + return True + except Exception as e: + logger.error(f"Failed to register DATA object {objectKey}: {e}") + return False + def registerFeatureDefinition(self, featureCode: str, label: Dict[str, str], icon: str) -> bool: """Register a feature definition.""" try: @@ -90,9 +114,23 @@ class RbacCatalogService: return [obj for obj in self._resourceObjects.values() if obj["featureCode"] == featureCode] return list(self._resourceObjects.values()) + def getDataObjects(self, featureCode: Optional[str] = None) -> List[Dict[str, Any]]: + """Get all DATA objects (tables/entities), optionally filtered by feature.""" + if featureCode: + return [obj for obj in self._dataObjects.values() if obj["featureCode"] == featureCode] + return list(self._dataObjects.values()) + def getAllObjects(self, featureCode: Optional[str] = None) -> List[Dict[str, Any]]: - """Get all RBAC objects (UI + RESOURCE), optionally filtered by feature.""" - return self.getUiObjects(featureCode) + self.getResourceObjects(featureCode) + """Get all RBAC objects (UI + RESOURCE + DATA), optionally filtered by feature.""" + return self.getUiObjects(featureCode) + self.getResourceObjects(featureCode) + self.getDataObjects(featureCode) + + def getAllCatalogObjects(self, featureCode: Optional[str] = None) -> Dict[str, List[Dict[str, Any]]]: + """Get all catalog objects grouped by type (DATA, UI, RESOURCE).""" + return { + "DATA": self.getDataObjects(featureCode), + "UI": self.getUiObjects(featureCode), + "RESOURCE": self.getResourceObjects(featureCode) + } def getFeatureDefinitions(self) -> List[Dict[str, Any]]: """Get all registered feature definitions.""" @@ -121,6 +159,8 @@ class RbacCatalogService: del self._uiObjects[key] for key in [k for k, v in self._resourceObjects.items() if v["featureCode"] == featureCode]: del self._resourceObjects[key] + for key in [k for k, v in self._dataObjects.items() if v["featureCode"] == featureCode]: + del self._dataObjects[key] self._featureDefinitions.pop(featureCode, None) self._templateRoles.pop(featureCode, None) logger.info(f"Unregistered feature: {featureCode}") @@ -135,6 +175,7 @@ class RbacCatalogService: "features": len(self._featureDefinitions), "uiObjects": len(self._uiObjects), "resourceObjects": len(self._resourceObjects), + "dataObjects": len(self._dataObjects), "templateRoles": sum(len(roles) for roles in self._templateRoles.values()) } diff --git a/modules/system/__init__.py b/modules/system/__init__.py new file mode 100644 index 00000000..7c14ddfa --- /dev/null +++ b/modules/system/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +System Module - Contains system-level infrastructure: +- registry.py: Feature container discovery and loading +- mainSystem.py: System-level RBAC objects (UI, DATA, RESOURCE) +""" + +from modules.system.registry import ( + loadFeatureRouters, + loadFeatureMainModules, + discoverFeatureContainers, + registerAllFeaturesInCatalog, +) diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py new file mode 100644 index 00000000..cbb33a52 --- /dev/null +++ b/modules/system/mainSystem.py @@ -0,0 +1,423 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +System Module - Main Module. +Registers system-level RBAC objects (UI, DATA, RESOURCE) that are not part of any feature. +These are global system pages and resources available to all users with appropriate roles. + +Also defines the navigation structure for the frontend. +""" + +import logging +from typing import Dict, List, Any, Optional + +logger = logging.getLogger(__name__) + +# System metadata +FEATURE_CODE = "system" +FEATURE_LABEL = {"en": "System", "de": "System", "fr": "Système"} +FEATURE_ICON = "mdi-cog" + +# ============================================================================= +# Navigation Structure (Single Source of Truth) +# ============================================================================= +# +# Block Order (gemäss Navigation-API-Konzept): +# - System: 10 +# - : 15 (wird in routeSystem.py eingefügt) +# - Workflows: 20 +# - Basisdaten: 30 +# - Migrate: 40 +# - Administration: 200 +# +# Item Order: Default-Abstand 10 pro Item +# uiComponent: Abgeleitet von objectKey (ui.system.home -> page.system.home) +# icon: Wird intern gehalten aber NICHT in der API Response zurückgegeben + +NAVIGATION_SECTIONS = [ + { + "id": "system", + "title": {"en": "SYSTEM", "de": "SYSTEM", "fr": "SYSTÈME"}, + "order": 10, + "items": [ + { + "id": "home", + "objectKey": "ui.system.home", + "label": {"en": "Home", "de": "Übersicht", "fr": "Accueil"}, + "icon": "FaHome", + "path": "/", + "order": 10, + "public": True, + }, + { + "id": "settings", + "objectKey": "ui.system.settings", + "label": {"en": "Settings", "de": "Einstellungen", "fr": "Paramètres"}, + "icon": "FaCog", + "path": "/settings", + "order": 20, + "public": True, + }, + ], + }, + { + "id": "workflows", + "title": {"en": "WORKFLOWS", "de": "WORKFLOWS", "fr": "WORKFLOWS"}, + "order": 20, + "items": [ + { + "id": "playground", + "objectKey": "ui.system.playground", + "label": {"en": "Chat Playground", "de": "Chat Playground", "fr": "Chat Playground"}, + "icon": "FaPlay", + "path": "/workflows/playground", + "order": 10, + }, + { + "id": "chats", + "objectKey": "ui.system.chats", + "label": {"en": "Chats", "de": "Chats", "fr": "Chats"}, + "icon": "FaListAlt", + "path": "/workflows/list", + "order": 20, + }, + { + "id": "automations", + "objectKey": "ui.system.automations", + "label": {"en": "Automations", "de": "Automatisierungen", "fr": "Automatisations"}, + "icon": "FaCogs", + "path": "/workflows/automations", + "order": 30, + }, + ], + }, + { + "id": "basedata", + "title": {"en": "BASE DATA", "de": "BASISDATEN", "fr": "DONNÉES DE BASE"}, + "order": 30, + "items": [ + { + "id": "prompts", + "objectKey": "ui.system.prompts", + "label": {"en": "Prompts", "de": "Prompts", "fr": "Prompts"}, + "icon": "FaLightbulb", + "path": "/basedata/prompts", + "order": 10, + }, + { + "id": "files", + "objectKey": "ui.system.files", + "label": {"en": "Files", "de": "Dateien", "fr": "Fichiers"}, + "icon": "FaRegFileAlt", + "path": "/basedata/files", + "order": 20, + }, + { + "id": "connections", + "objectKey": "ui.system.connections", + "label": {"en": "Connections", "de": "Verbindungen", "fr": "Connexions"}, + "icon": "FaLink", + "path": "/basedata/connections", + "order": 30, + }, + ], + }, + { + "id": "migrate", + "title": {"en": "MIGRATE TO FEATURES", "de": "MIGRATE TO FEATURES", "fr": "MIGRER VERS FEATURES"}, + "order": 40, + "deprecated": True, + "items": [ + { + "id": "chatbot", + "objectKey": "ui.system.chatbot", + "label": {"en": "Chatbot", "de": "Chatbot", "fr": "Chatbot"}, + "icon": "FaComments", + "path": "/chatbot", + "order": 10, + "deprecated": True, + }, + { + "id": "pek", + "objectKey": "ui.system.pek", + "label": {"en": "PEK", "de": "PEK", "fr": "PEK"}, + "icon": "FaChartBar", + "path": "/pek", + "order": 20, + "deprecated": True, + }, + { + "id": "speech", + "objectKey": "ui.system.speech", + "label": {"en": "Speech", "de": "Sprache", "fr": "Parole"}, + "icon": "FaMicrophone", + "path": "/speech", + "order": 30, + "deprecated": True, + }, + ], + }, + { + "id": "admin", + "title": {"en": "ADMINISTRATION", "de": "ADMINISTRATION", "fr": "ADMINISTRATION"}, + "order": 200, + "adminOnly": True, + "items": [ + { + "id": "admin-users", + "objectKey": "ui.admin.users", + "label": {"en": "Users", "de": "Benutzer", "fr": "Utilisateurs"}, + "icon": "FaUsers", + "path": "/admin/users", + "order": 10, + "adminOnly": True, + }, + { + "id": "admin-invitations", + "objectKey": "ui.admin.invitations", + "label": {"en": "Invitations", "de": "Einladungen", "fr": "Invitations"}, + "icon": "FaEnvelopeOpenText", + "path": "/admin/invitations", + "order": 20, + "adminOnly": True, + }, + { + "id": "admin-mandates", + "objectKey": "ui.admin.mandates", + "label": {"en": "Mandates", "de": "Mandanten", "fr": "Mandats"}, + "icon": "FaBuilding", + "path": "/admin/mandates", + "order": 30, + "adminOnly": True, + }, + { + "id": "admin-roles", + "objectKey": "ui.admin.roles", + "label": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, + "icon": "FaKey", + "path": "/admin/mandate-roles", + "order": 40, + "adminOnly": True, + }, + { + "id": "admin-role-permissions", + "objectKey": "ui.admin.role-permissions", + "label": {"en": "Role Permissions", "de": "Rollen-Berechtigungen", "fr": "Permissions des rôles"}, + "icon": "FaShieldAlt", + "path": "/admin/mandate-role-permissions", + "order": 50, + "adminOnly": True, + }, + { + "id": "admin-user-mandates", + "objectKey": "ui.admin.user-mandates", + "label": {"en": "Mandate Members", "de": "Mandanten-Mitglieder", "fr": "Membres du mandat"}, + "icon": "FaUserTag", + "path": "/admin/user-mandates", + "order": 60, + "adminOnly": True, + }, + { + "id": "admin-feature-roles", + "objectKey": "ui.admin.feature-roles", + "label": {"en": "Feature Roles & Permissions", "de": "Feature Rollen & Rechte", "fr": "Rôles et permissions des features"}, + "icon": "FaCube", + "path": "/admin/feature-roles", + "order": 70, + "adminOnly": True, + }, + { + "id": "admin-feature-instances", + "objectKey": "ui.admin.feature-instances", + "label": {"en": "Feature Instances", "de": "Feature-Instanzen", "fr": "Instances de feature"}, + "icon": "FaCubes", + "path": "/admin/feature-instances", + "order": 80, + "adminOnly": True, + }, + { + "id": "admin-feature-users", + "objectKey": "ui.admin.feature-users", + "label": {"en": "Feature Instance Users", "de": "Feature Instanz Benutzer", "fr": "Utilisateurs d'instance de feature"}, + "icon": "FaUsersCog", + "path": "/admin/feature-users", + "order": 90, + "adminOnly": True, + }, + ], + }, +] + + +def _objectKeyToUiComponent(objectKey: str) -> str: + """ + Convert objectKey to uiComponent. + + Example: ui.system.home -> page.system.home + ui.admin.users -> page.admin.users + ui.feature.trustee.dashboard -> page.feature.trustee.dashboard + """ + if objectKey.startswith("ui."): + return "page." + objectKey[3:] + return objectKey + + +def _buildUiObjectsFromNavigation() -> List[Dict[str, Any]]: + """Build UI_OBJECTS list from NAVIGATION_SECTIONS for RBAC registration.""" + uiObjects = [] + for section in NAVIGATION_SECTIONS: + for item in section.get("items", []): + uiObjects.append({ + "objectKey": item["objectKey"], + "label": item["label"], + "meta": { + "area": section["id"], + "public": item.get("public", False), + "adminOnly": item.get("adminOnly", False), + "deprecated": item.get("deprecated", False), + "path": item["path"], + "icon": item["icon"], + } + }) + return uiObjects + + +# Generate UI_OBJECTS from navigation structure +UI_OBJECTS = _buildUiObjectsFromNavigation() + +# ============================================================================= +# System DATA Objects +# ============================================================================= + +DATA_OBJECTS = [ + { + "objectKey": "data.system.User", + "label": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, + "meta": {"table": "UserInDB"} + }, + { + "objectKey": "data.system.Mandate", + "label": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, + "meta": {"table": "Mandate"} + }, + { + "objectKey": "data.system.Role", + "label": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, + "meta": {"table": "Role"} + }, + { + "objectKey": "data.system.AccessRule", + "label": {"en": "Access Rule", "de": "Zugriffsregel", "fr": "Règle d'accès"}, + "meta": {"table": "AccessRule"} + }, + { + "objectKey": "data.system.UserMandate", + "label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, + "meta": {"table": "UserMandate"} + }, + { + "objectKey": "data.system.Prompt", + "label": {"en": "Prompt", "de": "Prompt", "fr": "Prompt"}, + "meta": {"table": "Prompt"} + }, + { + "objectKey": "data.system.ChatWorkflow", + "label": {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de chat"}, + "meta": {"table": "ChatWorkflow"} + }, + { + "objectKey": "data.system.FileItem", + "label": {"en": "File", "de": "Datei", "fr": "Fichier"}, + "meta": {"table": "FileItem"} + }, + { + "objectKey": "data.system.UserConnection", + "label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"}, + "meta": {"table": "UserConnection"} + }, + { + "objectKey": "data.system.FeatureInstance", + "label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"}, + "meta": {"table": "FeatureInstance"} + }, +] + +# ============================================================================= +# System RESOURCE Objects +# ============================================================================= + +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.system.api.auth", + "label": {"en": "Authentication API", "de": "Authentifizierungs-API", "fr": "API d'authentification"}, + "meta": {"endpoint": "/api/auth/*"} + }, + { + "objectKey": "resource.system.api.users", + "label": {"en": "Users API", "de": "Benutzer-API", "fr": "API des utilisateurs"}, + "meta": {"endpoint": "/api/users/*"} + }, + { + "objectKey": "resource.system.api.mandates", + "label": {"en": "Mandates API", "de": "Mandanten-API", "fr": "API des mandats"}, + "meta": {"endpoint": "/api/mandates/*"} + }, + { + "objectKey": "resource.system.api.rbac", + "label": {"en": "RBAC API", "de": "RBAC-API", "fr": "API RBAC"}, + "meta": {"endpoint": "/api/rbac/*"} + }, +] + + +def registerFeature(catalogService) -> bool: + """ + Register system RBAC objects in the catalog. + + Args: + catalogService: The RBAC catalog service instance + + Returns: + True if registration was successful + """ + try: + # Register UI objects + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + # Register DATA objects + for dataObj in DATA_OBJECTS: + catalogService.registerDataObject( + featureCode=FEATURE_CODE, + objectKey=dataObj["objectKey"], + label=dataObj["label"], + meta=dataObj.get("meta") + ) + + # Register RESOURCE objects + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + # Register feature definition + catalogService.registerFeatureDefinition( + featureCode=FEATURE_CODE, + label=FEATURE_LABEL, + icon=FEATURE_ICON + ) + + logger.info(f"Registered system RBAC objects: {len(UI_OBJECTS)} UI, {len(DATA_OBJECTS)} DATA, {len(RESOURCE_OBJECTS)} RESOURCE") + return True + + except Exception as e: + logger.error(f"Failed to register system RBAC objects: {e}") + return False diff --git a/modules/features/featureRegistry.py b/modules/system/registry.py similarity index 82% rename from modules/features/featureRegistry.py rename to modules/system/registry.py index 4bf6d82e..5431b706 100644 --- a/modules/features/featureRegistry.py +++ b/modules/system/registry.py @@ -3,6 +3,8 @@ """ Feature Registry for Plug&Play Feature Container Loading. Dynamically discovers and loads feature containers from the features directory. + +Note: This module is in modules/system/ but manages modules/features/. """ import os @@ -14,8 +16,8 @@ from fastapi import FastAPI logger = logging.getLogger(__name__) -# Path to the features directory -FEATURES_DIR = os.path.dirname(os.path.abspath(__file__)) +# Path to the features directory (relative to this file's location) +FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "features") def discoverFeatureContainers() -> List[str]: @@ -109,10 +111,26 @@ def loadFeatureMainModules() -> Dict[str, Any]: def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: """ Register all features' RBAC objects in the catalog. + Also registers system-level RBAC objects. """ - mainModules = loadFeatureMainModules() results = {} + # Register system-level RBAC objects first + try: + from modules.system.mainSystem import registerFeature as registerSystemFeature + success = registerSystemFeature(catalogService) + results["system"] = success + if success: + logger.info("Registered RBAC objects: system") + except ImportError as e: + logger.warning(f"System module not found, skipping system RBAC registration: {e}") + except Exception as e: + logger.error(f"Error registering system RBAC objects: {e}") + results["system"] = False + + # Register feature modules + mainModules = loadFeatureMainModules() + for featureName, module in mainModules.items(): if hasattr(module, "registerFeature"): try: diff --git a/modules/system/routeSystem.py b/modules/system/routeSystem.py new file mode 100644 index 00000000..4e2f9f8f --- /dev/null +++ b/modules/system/routeSystem.py @@ -0,0 +1,515 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +System Routes - Navigation and system-level API endpoints. + +Navigation API Konzept: +- Single Source of Truth für Navigation im Gateway +- UI rendert nur was es erhält (keine Permission-Logik im UI) +- Keine Icons in API Response - UI mappt selbst via uiComponent +- Blocks statt Sections mit order auf allen Ebenen +""" + +import logging +from typing import Dict, List, Any, Optional +from fastapi import APIRouter, Depends, Request, Query +from slowapi import Limiter +from slowapi.util import get_remote_address + +from modules.auth.authentication import getRequestContext, RequestContext +from modules.system.mainSystem import NAVIGATION_SECTIONS, _objectKeyToUiComponent +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface +from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext +from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole, FeatureAccess, FeatureAccessRole + +logger = logging.getLogger(__name__) +limiter = Limiter(key_func=get_remote_address) + +# Main system router (for other system endpoints if needed) +router = APIRouter(prefix="/api/system", tags=["System"]) + +# Navigation router at /api/navigation (gemäss Navigation-API-Konzept) +navigationRouter = APIRouter(prefix="/api", tags=["Navigation"]) + + +def _getUserRoleIds(userId: str) -> List[str]: + """Get all role IDs for a user across all their mandates.""" + rootInterface = getRootInterface() + roleIds = [] + + userMandates = rootInterface.db.getRecordset( + UserMandate, + recordFilter={"userId": userId, "enabled": True} + ) + + for um in userMandates: + mandateRoleIds = rootInterface.getRoleIdsForUserMandate(um.get("id")) + for rid in mandateRoleIds: + if rid not in roleIds: + roleIds.append(rid) + + return roleIds + + +def _checkUiPermission(roleIds: List[str], objectKey: str) -> bool: + """Check if any of the given roles has view permission for the UI object.""" + if not roleIds: + return False + + rootInterface = getRootInterface() + + for roleId in roleIds: + # Get UI rules for this role + rules = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": roleId, "context": "UI"} + ) + + for rule in rules: + ruleItem = rule.get("item") + ruleView = rule.get("view", False) + + if not ruleView: + continue + + # Global rule (item=None) grants access to all UI + if ruleItem is None: + return True + + # Exact match + if ruleItem == objectKey: + return True + + # Wildcard match (e.g., ui.system.* matches ui.system.playground) + if ruleItem.endswith(".*"): + prefix = ruleItem[:-2] + if objectKey.startswith(prefix): + return True + + return False + + +# ============================================================================= +# Navigation API (gemäss Navigation-API-Konzept) +# Endpoint: GET /api/navigation +# ============================================================================= + +def _getFeatureUiObjects(featureCode: str) -> List[Dict[str, Any]]: + """ + Get UI objects for a feature from its main module. + Returns list of UI objects with objectKey, label, meta (including path). + """ + try: + # Dynamic import based on feature code + if featureCode == "trustee": + from modules.features.trustee.mainTrustee import UI_OBJECTS + return UI_OBJECTS + elif featureCode == "realestate": + from modules.features.realestate.mainRealEstate import UI_OBJECTS + return UI_OBJECTS + else: + logger.warning(f"Unknown feature code: {featureCode}") + return [] + except ImportError as e: + logger.error(f"Failed to import UI_OBJECTS for feature {featureCode}: {e}") + return [] + + +def _buildDynamicBlock( + userId: str, + language: str, + isSysAdmin: bool +) -> Optional[Dict[str, Any]]: + """ + Build the dynamic features block with mandates, features, and instances. + + Returns None if user has no feature instances. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Get all feature accesses for this user + featureAccesses = rootInterface.getFeatureAccessesForUser(userId) + + if not featureAccesses: + return None + + # Build hierarchical structure: mandate -> feature -> instances + mandatesMap: Dict[str, Dict[str, Any]] = {} + featuresMap: Dict[str, Dict[str, Any]] = {} # key: mandateId_featureCode + + mandateOrder = 10 + for access in featureAccesses: + if not access.enabled: + continue + + instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) + if not instance or not instance.enabled: + continue + + # Get mandate info + mandateId = str(instance.mandateId) + if mandateId not in mandatesMap: + mandate = rootInterface.getMandate(mandateId) + mandateName = mandate.name if mandate and hasattr(mandate, 'name') else mandateId + mandatesMap[mandateId] = { + "id": mandateId, + "uiLabel": mandateName, + "order": mandateOrder, + "features": [] + } + mandateOrder += 10 + + # Get feature info + featureKey = f"{mandateId}_{instance.featureCode}" + if featureKey not in featuresMap: + feature = featureInterface.getFeature(instance.featureCode) + + # Handle featureLabel - could be a dict or a Pydantic model (TextMultilingual) + if feature and hasattr(feature, 'label'): + featureLabel = feature.label + # Convert Pydantic model to dict if needed + if hasattr(featureLabel, 'model_dump'): + featureLabel = featureLabel.model_dump() + elif hasattr(featureLabel, 'dict'): + featureLabel = featureLabel.dict() + elif not isinstance(featureLabel, dict): + # Fallback: try to access as attributes + featureLabel = {"de": getattr(featureLabel, 'de', instance.featureCode), "en": getattr(featureLabel, 'en', instance.featureCode)} + else: + featureLabel = {"de": instance.featureCode, "en": instance.featureCode} + + featuresMap[featureKey] = { + "uiComponent": f"feature.{instance.featureCode}", + "uiLabel": featureLabel.get(language, featureLabel.get("en", instance.featureCode)), + "order": 10, + "instances": [], + "_mandateId": mandateId, + "_featureCode": instance.featureCode + } + + # Get user's permissions for this instance to filter views + permissions = _getInstanceViewPermissions(rootInterface, userId, str(instance.id), isSysAdmin) + + # Get feature UI objects to build views + featureUiObjects = _getFeatureUiObjects(instance.featureCode) + + # Build views for this instance + views = [] + viewOrder = 10 + for uiObj in featureUiObjects: + objectKey = uiObj.get("objectKey", "") + # Extract view name from objectKey for path building + viewName = objectKey.split(".")[-1] if objectKey else "" + + # Check permission using full objectKey (as per Navigation-API-Konzept) + if not isSysAdmin and not permissions.get("_all") and not permissions.get(objectKey, False): + continue + + # Skip admin-only views for non-admins + meta = uiObj.get("meta", {}) + if meta.get("admin_only") and not isSysAdmin and not permissions.get("isAdmin", False): + continue + + # Build path for this view + viewPath = f"/mandates/{mandateId}/{instance.featureCode}/{instance.id}/{viewName}" + + # Get label in requested language + label = uiObj.get("label", {}) + uiLabel = label.get(language, label.get("en", viewName)) + + views.append({ + "uiComponent": f"page.feature.{instance.featureCode}.{viewName}", + "uiLabel": uiLabel, + "uiPath": viewPath, + "order": viewOrder, + "objectKey": objectKey + }) + viewOrder += 10 + + # Sort views by order + views.sort(key=lambda v: v["order"]) + + # Add instance to feature + featuresMap[featureKey]["instances"].append({ + "id": str(instance.id), + "uiLabel": instance.label, + "order": 10, + "views": views + }) + + # Build final structure + for featureKey, featureData in featuresMap.items(): + mandateId = featureData.pop("_mandateId") + featureData.pop("_featureCode") + mandatesMap[mandateId]["features"].append(featureData) + + # Sort features within each mandate + for mandate in mandatesMap.values(): + mandate["features"].sort(key=lambda f: f["order"]) + + # Convert to list and sort by order + mandatesList = list(mandatesMap.values()) + mandatesList.sort(key=lambda m: m["order"]) + + if not mandatesList: + return None + + return { + "type": "dynamic", + "id": "features", + "title": "MEINE FEATURES", + "order": 15, # Between system (10) and workflows (20) + "mandates": mandatesList + } + + except Exception as e: + logger.error(f"Error building dynamic block: {e}") + return None + + +def _getInstanceViewPermissions( + rootInterface, + userId: str, + instanceId: str, + isSysAdmin: bool +) -> Dict[str, Any]: + """ + Get view permissions for a user in a feature instance. + Returns dict with view names as keys and True/False as values. + Also includes "_all" if user has global view access. + """ + if isSysAdmin: + return {"_all": True, "isAdmin": True} + + permissions = {"_all": False, "isAdmin": False} + + try: + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role + + # Get FeatureAccess for this user and instance + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) + + if not featureAccesses: + return permissions + + # Get role IDs via FeatureAccessRole junction table + featureAccessId = featureAccesses[0].get("id") + featureAccessRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + roleIds = [far.get("roleId") for far in featureAccessRoles] + + if not roleIds: + return permissions + + # Check if user has admin role + for roleId in roleIds: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles: + roleLabel = roles[0].get("roleLabel", "").lower() + if "admin" in roleLabel: + permissions["isAdmin"] = True + break + + # Get UI permissions from AccessRules + # Permissions are stored with full objectKey (e.g., ui.feature.trustee.dashboard) + for roleId in roleIds: + accessRules = rootInterface.db.getRecordset( + AccessRule, + recordFilter={"roleId": roleId, "context": "UI"} + ) + + logger.debug(f"_getInstanceViewPermissions: roleId={roleId}, UI rules count={len(accessRules)}") + + for rule in accessRules: + if not rule.get("view", False): + continue + + item = rule.get("item") + logger.debug(f"_getInstanceViewPermissions: rule item={item}, view={rule.get('view')}") + + if item is None: + # item=None means all views + permissions["_all"] = True + else: + # Store full objectKey as per Navigation-API-Konzept + permissions[item] = True + + logger.debug(f"_getInstanceViewPermissions: final permissions={permissions}") + return permissions + + except Exception as e: + logger.debug(f"Error getting instance view permissions: {e}") + return permissions + + +def _buildStaticBlocks( + language: str, + isSysAdmin: bool, + roleIds: List[str], + hasGlobalPermission: bool +) -> List[Dict[str, Any]]: + """ + Build static navigation blocks from NAVIGATION_SECTIONS. + + Returns list of blocks with items filtered by permissions. + """ + blocks = [] + + for section in NAVIGATION_SECTIONS: + # Skip admin-only sections for non-admins + if section.get("adminOnly") and not isSysAdmin: + continue + + # Filter items based on permissions + filteredItems = [] + for item in section.get("items", []): + # Skip admin-only items for non-admins + if item.get("adminOnly") and not isSysAdmin: + continue + + # Public items are always visible + if item.get("public"): + filteredItems.append(_formatBlockItem(item, language)) + continue + + # SysAdmin sees everything + if isSysAdmin: + filteredItems.append(_formatBlockItem(item, language)) + continue + + # Check permission for this item + if hasGlobalPermission or _checkUiPermission(roleIds, item["objectKey"]): + filteredItems.append(_formatBlockItem(item, language)) + + # Only include section if it has visible items + if filteredItems: + # Sort items by order + filteredItems.sort(key=lambda i: i["order"]) + + blocks.append({ + "type": "static", + "id": section["id"], + "title": section["title"].get(language, section["title"].get("en", section["id"])), + "order": section.get("order", 50), + "items": filteredItems, + }) + + return blocks + + +def _formatBlockItem(item: Dict[str, Any], language: str) -> Dict[str, Any]: + """ + Format a navigation item for the new API response. + + Uses new field names: uiComponent, uiLabel, uiPath + Does NOT include icon (UI maps via uiComponent) + """ + objectKey = item["objectKey"] + uiComponent = _objectKeyToUiComponent(objectKey) + + return { + "uiComponent": uiComponent, + "uiLabel": item["label"].get(language, item["label"].get("en", item["id"])), + "uiPath": item["path"], + "order": item.get("order", 50), + "objectKey": objectKey, + } + + +@navigationRouter.get("/navigation") +@limiter.limit("60/minute") +async def get_navigation( + request: Request, + language: str = Query("de", description="Language for labels (en, de, fr)"), + reqContext: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Get unified navigation structure with blocks. + + Single Source of Truth für Navigation - UI rendert nur was es erhält. + + Endpoint: GET /api/navigation + + Block order: + - System (10) + - Dynamic/Features (15) - only if user has feature instances + - Workflows (20) + - Basisdaten (30) + - Migrate (40) + - Administration (200) + + Response format: + { + "language": "de", + "blocks": [ + { + "type": "static", + "id": "system", + "title": "SYSTEM", + "order": 10, + "items": [ + { + "uiComponent": "page.system.home", + "uiLabel": "Übersicht", + "uiPath": "/", + "order": 10, + "objectKey": "ui.system.home" + } + ] + }, + { + "type": "dynamic", + "id": "features", + "title": "MEINE FEATURES", + "order": 15, + "mandates": [...] + } + ] + } + """ + try: + isSysAdmin = reqContext.isSysAdmin + userId = str(reqContext.user.id) if reqContext.user else None + + # Get user's role IDs for permission checking + roleIds = [] + if userId and not isSysAdmin: + roleIds = _getUserRoleIds(userId) + + # Check if user has global UI permission + hasGlobalPermission = isSysAdmin + if not hasGlobalPermission and roleIds: + hasGlobalPermission = _checkUiPermission(roleIds, "_global_check") + + # Build static blocks from NAVIGATION_SECTIONS + blocks = _buildStaticBlocks(language, isSysAdmin, roleIds, hasGlobalPermission) + + # Build dynamic block (features) if user has feature instances + if userId: + dynamicBlock = _buildDynamicBlock(userId, language, isSysAdmin) + if dynamicBlock: + blocks.append(dynamicBlock) + + # Sort all blocks by order + blocks.sort(key=lambda b: b["order"]) + + return { + "language": language, + "blocks": blocks, + } + + except Exception as e: + logger.error(f"Error getting navigation: {e}") + return { + "language": language, + "blocks": [], + "error": str(e), + } diff --git a/scripts/script_db_migrate_accessrules_objectkeys.py b/scripts/script_db_migrate_accessrules_objectkeys.py new file mode 100644 index 00000000..840367e5 --- /dev/null +++ b/scripts/script_db_migrate_accessrules_objectkeys.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Migration Script: Migrate AccessRules to Vollqualifizierte ObjectKeys + +This script migrates existing AccessRules in the database from short item names +(e.g., "dashboard", "positions") to fully qualified ObjectKeys +(e.g., "ui.feature.trustee.dashboard", "ui.feature.trustee.positions"). + +This is required for the Navigation-API-Konzept implementation. + +Usage: + python script_db_migrate_accessrules_objectkeys.py [--dry-run] + +Options: + --dry-run Show what would be changed without making actual changes +""" + +import sys +import os +import logging +from typing import Dict, List, Any, Optional + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +# Mapping of short item names to fully qualified ObjectKeys per feature +MIGRATION_MAP: Dict[str, Dict[str, str]] = { + "trustee": { + # UI items + "dashboard": "ui.feature.trustee.dashboard", + "positions": "ui.feature.trustee.positions", + "documents": "ui.feature.trustee.documents", + "position-documents": "ui.feature.trustee.position-documents", + "instance-roles": "ui.feature.trustee.instance-roles", + # RESOURCE items + "instance-roles.manage": "resource.feature.trustee.instance-roles.manage", + }, + "realestate": { + # UI items + "projects": "ui.feature.realestate.projects", + "parcels": "ui.feature.realestate.parcels", + # RESOURCE items + "project.create": "resource.feature.realestate.project.create", + "project.delete": "resource.feature.realestate.project.delete", + }, +} + + +def migrateAccessRules(dryRun: bool = False) -> Dict[str, int]: + """ + Migrate AccessRules from short item names to fully qualified ObjectKeys. + + Args: + dryRun: If True, don't make actual changes, just show what would be done + + Returns: + Dictionary with migration statistics + """ + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + stats = { + "total_rules": 0, + "migrated": 0, + "already_qualified": 0, + "skipped_null": 0, + "errors": 0, + } + + try: + rootInterface = getRootInterface() + db = rootInterface.db + + # Get all AccessRules + allRules = db.getRecordset(AccessRule, recordFilter=None) + stats["total_rules"] = len(allRules) + + logger.info(f"Found {len(allRules)} AccessRules to check") + + for rule in allRules: + ruleId = rule.get("id") + context = rule.get("context") + item = rule.get("item") + roleId = rule.get("roleId") + + # Skip rules without item (wildcard rules) + if item is None: + stats["skipped_null"] += 1 + continue + + # Skip if already fully qualified + if item.startswith("ui.") or item.startswith("resource.") or item.startswith("data."): + stats["already_qualified"] += 1 + continue + + # Get the role to determine feature code + roles = db.getRecordset(Role, recordFilter={"id": roleId}) + if not roles or len(roles) == 0: + logger.warning(f"Rule {ruleId}: Role {roleId} not found, skipping") + stats["errors"] += 1 + continue + + role = roles[0] + featureCode = role.get("featureCode") + if not featureCode or featureCode not in MIGRATION_MAP: + logger.debug(f"Rule {ruleId}: Feature '{featureCode}' not in migration map, skipping") + continue + + # Lookup the new ObjectKey + featureMap = MIGRATION_MAP[featureCode] + if item not in featureMap: + logger.warning(f"Rule {ruleId}: Item '{item}' not in migration map for feature '{featureCode}'") + stats["errors"] += 1 + continue + + newItem = featureMap[item] + + if dryRun: + logger.info(f"[DRY-RUN] Would migrate rule {ruleId}: '{item}' -> '{newItem}' (feature: {featureCode})") + else: + # Update the rule using recordModify + try: + db.recordModify(AccessRule, ruleId, {"item": newItem}) + logger.info(f"Migrated rule {ruleId}: '{item}' -> '{newItem}' (feature: {featureCode})") + except Exception as e: + logger.error(f"Failed to migrate rule {ruleId}: {e}") + stats["errors"] += 1 + continue + + stats["migrated"] += 1 + + return stats + + except Exception as e: + logger.error(f"Migration failed: {e}", exc_info=True) + raise + + +def main(): + """Main entry point.""" + dryRun = "--dry-run" in sys.argv + forceRun = "--force" in sys.argv + + if dryRun: + logger.info("=" * 60) + logger.info("DRY RUN MODE - No changes will be made") + logger.info("=" * 60) + else: + logger.info("=" * 60) + logger.info("LIVE MODE - Changes will be applied to the database") + logger.info("=" * 60) + + if not forceRun: + # Confirm before proceeding + confirm = input("Are you sure you want to proceed? (yes/no): ") + if confirm.lower() != "yes": + logger.info("Migration cancelled") + return + + try: + stats = migrateAccessRules(dryRun=dryRun) + + logger.info("=" * 60) + logger.info("Migration completed!") + logger.info(f" Total rules checked: {stats['total_rules']}") + logger.info(f" Rules migrated: {stats['migrated']}") + logger.info(f" Already qualified: {stats['already_qualified']}") + logger.info(f" Skipped (null item): {stats['skipped_null']}") + logger.info(f" Errors: {stats['errors']}") + logger.info("=" * 60) + + except Exception as e: + logger.error(f"Migration failed: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/script_export_accessrules.py b/scripts/script_export_accessrules.py new file mode 100644 index 00000000..6d5aeec7 --- /dev/null +++ b/scripts/script_export_accessrules.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Export Script: Generate Access Rules per Role Report + +Usage: + python script_export_accessrules.py > output.md + python script_export_accessrules.py --file ../docs/reports/access-rules.md +""" + +import sys +import os +from datetime import datetime + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.datamodels.datamodelRbac import Role, AccessRule + +def main(): + rootInterface = getRootInterface() + db = rootInterface.db + + # Get all roles and rules + roles = db.getRecordset(Role, recordFilter=None) + rules = db.getRecordset(AccessRule, recordFilter=None) + + # Group rules by role + rulesByRole = {} + for rule in rules: + roleId = rule.get('roleId') + if roleId not in rulesByRole: + rulesByRole[roleId] = [] + rulesByRole[roleId].append(rule) + + # Build markdown + lines = [] + lines.append('# Access Rules per Role') + lines.append('') + lines.append(f'Generated: {datetime.now().isoformat()}') + lines.append('') + lines.append(f'Total Roles: {len(roles)}') + lines.append(f'Total Rules: {len(rules)}') + lines.append('') + lines.append('---') + lines.append('') + + for role in sorted(roles, key=lambda r: (r.get('featureCode') or '', r.get('code') or '')): + roleId = role.get('id') + roleName = role.get('name') or role.get('code') + featureCode = role.get('featureCode') or 'system' + roleRules = rulesByRole.get(roleId, []) + + lines.append(f'## {featureCode} / {roleName}') + lines.append('') + lines.append(f'- **Role ID:** `{roleId}`') + lines.append(f'- **Code:** `{role.get("code")}`') + lines.append(f'- **Feature:** `{featureCode}`') + lines.append(f'- **Rules Count:** {len(roleRules)}') + lines.append('') + + if roleRules: + lines.append('| Context | Item | Access |') + lines.append('|---------|------|--------|') + for rule in sorted(roleRules, key=lambda r: (r.get('context') or '', r.get('item') or '')): + ctx = rule.get('context') or '*' + item = rule.get('item') or '*' + access = rule.get('access') or 'allow' + lines.append(f'| {ctx} | `{item}` | {access} |') + lines.append('') + else: + lines.append('*No rules defined*') + lines.append('') + + lines.append('---') + lines.append('') + + md = '\n'.join(lines) + + # Check for --file argument + if '--file' in sys.argv: + idx = sys.argv.index('--file') + if idx + 1 < len(sys.argv): + filePath = sys.argv[idx + 1] + with open(filePath, 'w', encoding='utf-8') as f: + f.write(md) + print(f'Written to {filePath}') + return + + # Output to stdout + sys.stdout.reconfigure(encoding='utf-8') + print(md) + +if __name__ == "__main__": + main() From bc2877bcc16b47fd8ae8a12c14779f11b885823f Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 25 Jan 2026 03:18:22 +0100 Subject: [PATCH 24/32] fixed rbac feature rules to override global rules, not to combine --- .../trustee/interfaceFeatureTrustee.py | 168 +----------------- modules/security/rbac.py | 20 ++- 2 files changed, 23 insertions(+), 165 deletions(-) diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index 729793b4..bdea38c1 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -663,9 +663,6 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Step 2: Feature-level filtering based on trustee.access - records = self.filterRecordsByTrusteeAccess(records, TrusteeContract) - totalItems = len(records) if params: pageSize = params.pageSize or 20 @@ -698,10 +695,7 @@ class TrusteeObjects: mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) - - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(records, TrusteeContract) - return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] def updateContract(self, contractId: str, data: Dict[str, Any]) -> Optional[TrusteeContract]: """Update a contract (organisationId is immutable).""" @@ -808,10 +802,6 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Step 2: Feature-level filtering based on trustee.access - # This applies userreport filtering (only own records) - records = self.filterRecordsByTrusteeAccess(records, TrusteeDocument) - # Convert dicts to Pydantic objects (remove binary data and internal fields) pydanticItems = [] for record in records: @@ -851,11 +841,8 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(records, TrusteeDocument) - result = [] - for record in filtered: + for record in records: cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} result.append(TrusteeDocument(**cleanedRecord)) return result @@ -961,10 +948,6 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Step 2: Feature-level filtering based on trustee.access - # This applies userreport filtering (only own records) - records = self.filterRecordsByTrusteeAccess(records, TrusteePosition) - # Convert dicts to Pydantic objects (remove internal fields) pydanticItems = [] for record in records: @@ -1003,10 +986,7 @@ class TrusteeObjects: mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) - - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(records, TrusteePosition) - return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] def getPositionsByOrganisation(self, organisationId: str) -> List[TrusteePosition]: """Get all positions for a specific organisation.""" @@ -1020,10 +1000,7 @@ class TrusteeObjects: mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) - - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(records, TrusteePosition) - return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] def updatePosition(self, positionId: str, data: Dict[str, Any]) -> Optional[TrusteePosition]: """Update a position. @@ -1148,10 +1125,6 @@ class TrusteeObjects: enrichPermissions=True ) - # Step 2: Feature-level filtering based on trustee.access - # This applies userreport filtering (only own records) - records = self.filterRecordsByTrusteeAccess(records, TrusteePositionDocument) - totalItems = len(records) if params: pageSize = params.pageSize or 20 @@ -1184,10 +1157,7 @@ class TrusteeObjects: mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) - - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) - return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] def getPositionsForDocument(self, documentId: str) -> List[TrusteePositionDocument]: """Get all positions linked to a document.""" @@ -1201,10 +1171,7 @@ class TrusteeObjects: mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) - - # Step 2: Feature-level filtering based on trustee.access - filtered = self.filterRecordsByTrusteeAccess(links, TrusteePositionDocument) - return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] def deletePositionDocument(self, linkId: str) -> bool: """Delete a position-document link. @@ -1368,126 +1335,3 @@ class TrusteeObjects: return True return False - - def filterRecordsByTrusteeAccess( - self, - records: List[Dict[str, Any]], - modelClass: type - ) -> List[Dict[str, Any]]: - """ - Filter records based on user's trustee.access permissions. - - Args: - records: List of records to filter - modelClass: The model class for determining filter logic - - Returns: - Filtered list of records - """ - if not records: - return records - - # Users with ALL access level bypass feature-level filtering - accessLevel = self.getRbacAccessLevel(modelClass, "read") - if accessLevel == AccessLevel.ALL: - return records - - # NEW: Feature-instance based access (new system) - # If featureInstanceId is set, user has access via FeatureAccess system. - # Data is already filtered by featureInstanceId in getRecordsetWithRBAC. - # The old TrusteeAccess system (organisation-based) is not used for - # feature-instance scoped data. - if self.featureInstanceId: - return records # User already has access to this feature instance - - # LEGACY: TrusteeAccess based filtering (for backwards compatibility) - # Get all user's access records - userAccess = self.getAllUserAccess(self.userId) - - if not userAccess: - # No trustee access at all - return empty for trustee tables - return [] - - # Build lookup for user's accessible organisations and contracts - accessByOrg = {} # {orgId: {'roles': [...], 'contracts': [...]}} - hasFullOrgAccess = {} # {orgId: True} if user has access without contractId restriction - - for access in userAccess: - orgId = access.get("organisationId") - roleId = access.get("roleId") - contractIdAccess = access.get("contractId") - - if orgId not in accessByOrg: - accessByOrg[orgId] = {"roles": [], "contracts": []} - - if roleId not in accessByOrg[orgId]["roles"]: - accessByOrg[orgId]["roles"].append(roleId) - - if contractIdAccess is None: - hasFullOrgAccess[orgId] = True - elif contractIdAccess not in accessByOrg[orgId]["contracts"]: - accessByOrg[orgId]["contracts"].append(contractIdAccess) - - filteredRecords = [] - - for record in records: - orgId = record.get("organisationId") - contractId = record.get("contractId") - createdBy = record.get("_createdBy") - - # For Organisation model, filter by accessible organisations - if modelClass == TrusteeOrganisation: - recordOrgId = record.get("id") - if recordOrgId in accessByOrg: - filteredRecords.append(record) - continue - - # Check if user has access to this organisation - if orgId not in accessByOrg: - continue - - roles = accessByOrg[orgId]["roles"] - - # admin has full access to organisation - if "admin" in roles: - # Check contract filtering - if contractId: - if hasFullOrgAccess.get(orgId) or contractId in accessByOrg[orgId]["contracts"]: - filteredRecords.append(record) - else: - filteredRecords.append(record) - continue - - # operate has full access to organisation data - if "operate" in roles: - if contractId: - if hasFullOrgAccess.get(orgId) or contractId in accessByOrg[orgId]["contracts"]: - filteredRecords.append(record) - else: - filteredRecords.append(record) - continue - - # userreport can only see own records for documents/positions - if "userreport" in roles: - if modelClass in (TrusteeDocument, TrusteePosition, TrusteePositionDocument): - # Must be own record - if createdBy == self.userId: - # Also check contract access - if contractId: - if hasFullOrgAccess.get(orgId) or contractId in accessByOrg[orgId]["contracts"]: - filteredRecords.append(record) - else: - filteredRecords.append(record) - elif modelClass == TrusteeContract: - # Can read contracts in their organisation - if contractId: - if hasFullOrgAccess.get(orgId) or contractId in accessByOrg[orgId]["contracts"]: - filteredRecords.append(record) - else: - # For contracts table, check if record's id is in accessible contracts - recordContractId = record.get("id") - if hasFullOrgAccess.get(orgId) or recordContractId in accessByOrg[orgId]["contracts"]: - filteredRecords.append(record) - continue - - return filteredRecords diff --git a/modules/security/rbac.py b/modules/security/rbac.py index f236852a..d35120cb 100644 --- a/modules/security/rbac.py +++ b/modules/security/rbac.py @@ -111,14 +111,22 @@ class RbacClass: if roleId not in rolePermissions or priority > rolePermissions[roleId][0]: rolePermissions[roleId] = (priority, rule) - # Combine permissions across roles using opening (union) logic + # Find highest priority among matching rules + highestPriority = max((p for p, _ in rolePermissions.values()), default=0) + + # Combine permissions ONLY from rules with highest priority + # This ensures instance-specific rules (Priority 3) override global rules (Priority 1) for roleId, (priority, rule) in rolePermissions.items(): + # Only use rules with highest priority + if priority < highestPriority: + continue + # View: union logic - if ANY role has view=true, then view=true if rule.view: permissions.view = True if context == AccessRuleContext.DATA: - # For DATA context, use most permissive access level across roles + # For DATA context, use most permissive access level across roles at same priority if rule.read and self._isMorePermissive(rule.read, permissions.read): permissions.read = rule.read if rule.create and self._isMorePermissive(rule.create, permissions.create): @@ -375,7 +383,8 @@ class RbacClass: Args: rule: Access rule to check - item: Item to match against + item: Item to match against (can be short name like "TrusteePosition" or + fully qualified like "data.feature.trustee.TrusteePosition") Returns: True if rule matches item @@ -396,6 +405,11 @@ class RbacClass: if item.startswith(rule.item + "."): return True + # Suffix match: rule.item ends with ".{item}" (e.g., "data.feature.trustee.TrusteePosition" matches "TrusteePosition") + # This allows short table names to match fully qualified objectKeys + if rule.item.endswith("." + item): + return True + return False def findMostSpecificRule(self, rules: List[AccessRule], item: str) -> Optional[AccessRule]: From e737bf5cdba5f7566ce67eab2caecaa77a07a1ca Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 25 Jan 2026 23:57:41 +0100 Subject: [PATCH 25/32] gpdr compliancy implemented --- app.py | 7 +- .../automation/datamodelFeatureAutomation.py | 2 +- .../automation/routeFeatureAutomation.py | 43 +- .../automation/subAutomationTemplates.py | 32 + .../realEstate/interfaceFeatureRealEstate.py | 37 +- .../trustee/interfaceFeatureTrustee.py | 52 +- modules/features/trustee/mainTrustee.py | 6 + modules/interfaces/interfaceBootstrap.py | 88 +-- modules/interfaces/interfaceRbac.py | 32 +- modules/routes/routeAdminRbacExport.py | 4 +- .../routes/routeAdminUserAccessOverview.py | 503 ++++++++++++ modules/routes/routeGdpr.py | 98 +-- modules/routes/routeInvitations.py | 179 +---- modules/routes/routeSharepoint.py | 105 +++ modules/{system => routes}/routeSystem.py | 0 modules/security/rbac.py | 21 +- modules/shared/gdprDeletion.py | 679 ++++++++++++++++ modules/system/mainSystem.py | 52 +- .../methodSharepoint/actions/__init__.py | 2 + .../actions/getExpensesFromPdf.py | 733 ++++++++++++++++++ .../methodSharepoint/methodSharepoint.py | 38 + tests/unit/rbac/test_rbac_bootstrap.py | 24 +- tests/unit/rbac/test_rbac_permissions.py | 25 +- 23 files changed, 2386 insertions(+), 376 deletions(-) create mode 100644 modules/routes/routeAdminUserAccessOverview.py rename modules/{system => routes}/routeSystem.py (100%) create mode 100644 modules/shared/gdprDeletion.py create mode 100644 modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py diff --git a/app.py b/app.py index b78e5d8b..dec478fc 100644 --- a/app.py +++ b/app.py @@ -409,7 +409,7 @@ app.add_middleware( CORSMiddleware, allow_origins=getAllowedOrigins(), allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], allow_headers=["*"], expose_headers=["*"], max_age=86400, # Increased caching for preflight requests @@ -495,6 +495,9 @@ app.include_router(invitationsRouter) from modules.routes.routeAdminRbacExport import router as rbacAdminExportRouter app.include_router(rbacAdminExportRouter) +from modules.routes.routeAdminUserAccessOverview import router as userAccessOverviewRouter +app.include_router(userAccessOverviewRouter) + from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) @@ -504,7 +507,7 @@ app.include_router(chatRouter) # ============================================================================ # SYSTEM ROUTES (Navigation, etc.) # ============================================================================ -from modules.system.routeSystem import router as systemRouter, navigationRouter +from modules.routes.routeSystem import router as systemRouter, navigationRouter app.include_router(systemRouter) app.include_router(navigationRouter) diff --git a/modules/features/automation/datamodelFeatureAutomation.py b/modules/features/automation/datamodelFeatureAutomation.py index 2a774ebd..7988cadf 100644 --- a/modules/features/automation/datamodelFeatureAutomation.py +++ b/modules/features/automation/datamodelFeatureAutomation.py @@ -19,7 +19,7 @@ class AutomationDefinition(BaseModel): {"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}} ]}) template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True}) - placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"}) + placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "textarea"}) active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False}) eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 4eef9381..c92200de 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -14,7 +14,7 @@ import json # Import interfaces and models from modules.interfaces.interfaceDbChat import getInterface as getChatInterface -from modules.auth import getCurrentUser, limiter, getRequestContext, RequestContext +from modules.auth import limiter, getRequestContext, RequestContext from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition from modules.datamodels.datamodelChat import ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict @@ -210,6 +210,47 @@ async def update_automation( detail=f"Error updating automation: {str(e)}" ) +@router.patch("/{automationId}/status") +@limiter.limit("30/minute") +async def update_automation_status( + request: Request, + automationId: str = Path(..., description="Automation ID"), + active: bool = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext) +) -> AutomationDefinition: + """Update only the active status of an automation definition""" + try: + chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None) + + # Get existing automation + automation = chatInterface.getAutomationDefinition(automationId) + if not automation: + raise HTTPException( + status_code=404, + detail=f"Automation {automationId} not found" + ) + + # Update only the active field + automationData = automation if isinstance(automation, dict) else automation.model_dump() + automationData['active'] = active + + updated = chatInterface.updateAutomationDefinition(automationId, automationData) + return updated + except HTTPException: + raise + except PermissionError as e: + raise HTTPException( + status_code=403, + detail=str(e) + ) + except Exception as e: + logger.error(f"Error updating automation status: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error updating automation status: {str(e)}" + ) + + @router.delete("/{automationId}") @limiter.limit("10/minute") async def delete_automation( diff --git a/modules/features/automation/subAutomationTemplates.py b/modules/features/automation/subAutomationTemplates.py index 95c1eb77..e95ca04d 100644 --- a/modules/features/automation/subAutomationTemplates.py +++ b/modules/features/automation/subAutomationTemplates.py @@ -369,6 +369,38 @@ AUTOMATION_TEMPLATES: Dict[str, Any] = { "jiraIssueType": "Task", "taskSyncDefinition": "{\"ID\":[\"get\",[\"key\"]],\"Module Category\":[\"get\",[\"fields\",\"customfield_10058\",\"value\"]],\"Summary\":[\"get\",[\"fields\",\"summary\"]],\"Description\":[\"get\",[\"fields\",\"description\"]],\"References\":[\"get\",[\"fields\",\"customfield_10066\"]],\"Priority\":[\"get\",[\"fields\",\"priority\",\"name\"]],\"Issue Status\":[\"get\",[\"fields\",\"status\",\"name\"]],\"Assignee\":[\"get\",[\"fields\",\"assignee\",\"displayName\"]],\"Issue Created\":[\"get\",[\"fields\",\"created\"]],\"Due Date\":[\"get\",[\"fields\",\"duedate\"]],\"DELTA Comments\":[\"get\",[\"fields\",\"customfield_10167\"]],\"SELISE Ticket References\":[\"put\",[\"fields\",\"customfield_10067\"]],\"SELISE Status Values\":[\"put\",[\"fields\",\"customfield_10065\"]],\"SELISE Comments\":[\"put\",[\"fields\",\"customfield_10168\"]]}" } + }, + { + "template": { + "overview": "Expenses PDF to Trustee Position", + "tasks": [ + { + "id": "Task01", + "title": "Extract Expenses from SharePoint PDFs", + "description": "Reads PDF expense documents from SharePoint folder, extracts data via AI, and saves to TrusteePosition", + "objective": "Extract expense data from PDF documents and store in Trustee database with automatic file organization", + "actionList": [ + { + "execMethod": "sharepoint", + "execAction": "getExpensesFromPdf", + "execParameters": { + "connectionReference": "{{KEY:connectionName}}", + "sharepointFolder": "{{KEY:sharepointFolder}}", + "featureInstanceId": "{{KEY:featureInstanceId}}", + "prompt": "{{KEY:extractionPrompt}}" + }, + "execResultLabel": "expense_extraction_result" + } + ] + } + ] + }, + "parameters": { + "connectionName": "", + "sharepointFolder": "", + "featureInstanceId": "", + "extractionPrompt": "Du bist ein Spezialist für die Extraktion von Spesendaten aus PDF-Dokumenten.\n\nAUFGABE:\nExtrahiere alle Speseneinträge aus dem bereitgestellten PDF-Dokument und gib sie im CSV-Format zurück.\n\nWICHTIGE REGELN:\n1. Pro MwSt-Prozentsatz einen separaten Datensatz erstellen\n2. Alle Datensätze zusammen müssen den Gesamtbetrag des Dokuments ergeben\n3. Der gesamte extrahierte Text des Dokuments muss im Feld \"desc\" erfasst werden\n4. Feld \"company\" enthält den Lieferanten/Verkäufer der Buchung\n5. Tags müssen aus dieser Liste gewählt werden: customer, meeting, license, subscription, fuel, food, material\n - Mehrere zutreffende Tags mit Komma trennen\n\nCSV-SPALTEN (in dieser Reihenfolge):\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n\nDATENFORMAT:\n- valuta: YYYY-MM-DD (Valutadatum)\n- transactionDateTime: Unix-Timestamp in Sekunden (Transaktionszeitpunkt)\n- company: Lieferant/Verkäufer Name\n- desc: Vollständiger extrahierter Text des Dokuments\n- tags: Komma-getrennte Tags aus der erlaubten Liste\n- bookingCurrency: Währungscode (CHF, EUR, USD, GBP)\n- bookingAmount: Buchungsbetrag als Dezimalzahl\n- originalCurrency: Original-Währungscode\n- originalAmount: Original-Betrag als Dezimalzahl\n- vatPercentage: MwSt-Prozentsatz (z.B. 8.1 für 8.1%)\n- vatAmount: MwSt-Betrag als Dezimalzahl\n\nBEISPIEL OUTPUT:\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",food,CHF,45.50,CHF,45.50,2.6,1.15\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",material,CHF,12.30,CHF,12.30,8.1,0.92\n\nHINWEISE:\n- Wenn nur ein MwSt-Satz vorhanden ist, einen Datensatz erstellen\n- Wenn mehrere MwSt-Sätze vorhanden sind (z.B. Lebensmittel 2.6% und Non-Food 8.1%), separate Datensätze erstellen\n- Bei fehlenden Informationen: leeres Feld oder Standardwert\n- Keine Anführungszeichen um numerische Werte" + } } ] } diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 7a96afaa..40a85c7e 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -39,6 +39,10 @@ class RealEstateObjects: Handles CRUD operations on Real Estate entities. """ + # Feature code for RBAC objectKey construction + # Used to build: data.feature.realestate.{TableName} + FEATURE_CODE = "realestate" + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Real Estate Interface. @@ -167,7 +171,8 @@ class RealEstateObjects: self.db, Projekt, self.currentUser, - recordFilter={"id": projektId} + recordFilter={"id": projektId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -181,7 +186,8 @@ class RealEstateObjects: self.db, Projekt, self.currentUser, - recordFilter=recordFilter or {} + recordFilter=recordFilter or {}, + featureCode=self.FEATURE_CODE ) return [Projekt(**r) for r in records] @@ -255,7 +261,8 @@ class RealEstateObjects: self.db, Parzelle, self.currentUser, - recordFilter={"id": parzelleId} + recordFilter={"id": parzelleId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -464,7 +471,8 @@ class RealEstateObjects: self.db, Dokument, self.currentUser, - recordFilter={"id": dokumentId} + recordFilter={"id": dokumentId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -478,7 +486,8 @@ class RealEstateObjects: self.db, Dokument, self.currentUser, - recordFilter=recordFilter or {} + recordFilter=recordFilter or {}, + featureCode=self.FEATURE_CODE ) return [Dokument(**r) for r in records] @@ -533,7 +542,8 @@ class RealEstateObjects: self.db, Gemeinde, self.currentUser, - recordFilter={"id": gemeindeId} + recordFilter={"id": gemeindeId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -547,7 +557,8 @@ class RealEstateObjects: self.db, Gemeinde, self.currentUser, - recordFilter=recordFilter or {} + recordFilter=recordFilter or {}, + featureCode=self.FEATURE_CODE ) return [Gemeinde(**r) for r in records] @@ -602,7 +613,8 @@ class RealEstateObjects: self.db, Kanton, self.currentUser, - recordFilter={"id": kantonId} + recordFilter={"id": kantonId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -616,7 +628,8 @@ class RealEstateObjects: self.db, Kanton, self.currentUser, - recordFilter=recordFilter or {} + recordFilter=recordFilter or {}, + featureCode=self.FEATURE_CODE ) return [Kanton(**r) for r in records] @@ -671,7 +684,8 @@ class RealEstateObjects: self.db, Land, self.currentUser, - recordFilter={"id": landId} + recordFilter={"id": landId}, + featureCode=self.FEATURE_CODE ) if not records: @@ -685,7 +699,8 @@ class RealEstateObjects: self.db, Land, self.currentUser, - recordFilter=recordFilter or {} + recordFilter=recordFilter or {}, + featureCode=self.FEATURE_CODE ) return [Land(**r) for r in records] diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index bdea38c1..df8038f9 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -66,6 +66,10 @@ class TrusteeObjects: Interface to Trustee database. Manages trustee organisations, roles, access, contracts, documents, and positions. """ + + # Feature code for RBAC objectKey construction + # Used to build: data.feature.trustee.{TableName} + FEATURE_CODE = "trustee" def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Trustee Interface. @@ -173,7 +177,8 @@ class TrusteeObjects: AccessRuleContext.DATA, tableName, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) if not permissions.view: @@ -200,7 +205,8 @@ class TrusteeObjects: AccessRuleContext.DATA, tableName, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) if not permissions.view: @@ -264,7 +270,8 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) logger.debug(f"getAllOrganisations: getRecordsetWithRBAC returned {len(records)} records") @@ -357,7 +364,8 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) # Users with ALL access level (from system RBAC) see all roles @@ -467,7 +475,8 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) # Users with ALL access level (from system RBAC) see all records @@ -526,7 +535,8 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -543,7 +553,8 @@ class TrusteeObjects: recordFilter={"userId": userId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) # Users with ALL access level (from system RBAC) see all records @@ -660,7 +671,8 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) totalItems = len(records) @@ -693,7 +705,8 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="label", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -799,7 +812,8 @@ class TrusteeObjects: recordFilter=None, orderBy="documentName", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) # Convert dicts to Pydantic objects (remove binary data and internal fields) @@ -838,7 +852,8 @@ class TrusteeObjects: recordFilter={"contractId": contractId}, orderBy="documentName", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) result = [] @@ -945,7 +960,8 @@ class TrusteeObjects: recordFilter=None, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) # Convert dicts to Pydantic objects (remove internal fields) @@ -984,7 +1000,8 @@ class TrusteeObjects: recordFilter={"contractId": contractId}, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -998,7 +1015,8 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -1155,7 +1173,8 @@ class TrusteeObjects: recordFilter={"positionId": positionId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] @@ -1169,7 +1188,8 @@ class TrusteeObjects: recordFilter={"documentId": documentId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode=self.FEATURE_CODE ) return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 311293f6..4f1694b5 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -38,6 +38,11 @@ UI_OBJECTS = [ "label": {"en": "Position Documents", "de": "Positions-Dokumente", "fr": "Documents de position"}, "meta": {"area": "position-documents"} }, + { + "objectKey": "ui.feature.trustee.expense-import", + "label": {"en": "Expense Import", "de": "Spesen Import", "fr": "Import de dépenses"}, + "meta": {"area": "expense-import"} + }, { "objectKey": "ui.feature.trustee.instance-roles", "label": {"en": "Instance Roles & Permissions", "de": "Instanz-Rollen & Berechtigungen", "fr": "Rôles et permissions d'instance"}, @@ -144,6 +149,7 @@ TEMPLATE_ROLES = [ {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, {"context": "UI", "item": "ui.feature.trustee.position-documents", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.expense-import", "view": True}, # Group-level DATA access {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, ] diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 472e77f9..0e66ce7b 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -376,13 +376,21 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: userId = _getRoleId(db, "user") viewerId = _getRoleId(db, "viewer") + # ========================================================================== + # SYSTEM TABLE RULES - Using standardized dot format: data.system.{TableName} + # ========================================================================== + # All DATA context items MUST use the full objectKey format for consistency. + # This matches the DATA_OBJECTS registration in mainSystem.py. + # Feature tables use: data.feature.{featureCode}.{TableName} + # ========================================================================== + # Mandate table - Only SysAdmin (flag) can access, not roles # Regular roles have no access to Mandate table if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="Mandate", + item="data.system.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -393,7 +401,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="Mandate", + item="data.system.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -404,7 +412,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="Mandate", + item="data.system.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -417,7 +425,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -428,7 +436,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -439,7 +447,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -447,26 +455,19 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # System tables only - NOT feature-specific tables! - # Feature tables (TrusteeXXX, Projekt, etc.) are handled by FEATURE-TEMPLATE roles. - # NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag - # - # Proper format: Just table names for DATA context (item="TableName") - # The full data.system.TableName format is for catalog registration only. - # FileItem and UserConnection: All users (user, admin, viewer) only MY-level CRUD restrictedTables = [ - "UserConnection", # User connections/sessions - only own records - "FileItem", # Uploaded files - only own files + "data.system.UserConnection", # User connections/sessions - only own records + "data.system.FileItem", # Uploaded files - only own files ] - for table in restrictedTables: + for objectKey in restrictedTables: # Admin: Only MY-level access (not group-level!) if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item=table, + item=objectKey, view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -478,7 +479,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item=table, + item=objectKey, view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -490,7 +491,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item=table, + item=objectKey, view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -501,37 +502,34 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: # Prompt: Special rule - CRUD for MY + Read for GROUP # Each user can manage own prompts (m) but can read group prompts (g) if adminId: - # Admin: MY-level CRUD + GROUP-level read tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="Prompt", + item="data.system.Prompt", view=True, - read=AccessLevel.GROUP, # Can read group prompts - create=AccessLevel.MY, # Can create own prompts - update=AccessLevel.MY, # Can update own prompts - delete=AccessLevel.MY, # Can delete own prompts + read=AccessLevel.GROUP, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, )) if userId: - # User: MY-level CRUD + GROUP-level read tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="Prompt", + item="data.system.Prompt", view=True, - read=AccessLevel.GROUP, # Can read group prompts - create=AccessLevel.MY, # Can create own prompts - update=AccessLevel.MY, # Can update own prompts - delete=AccessLevel.MY, # Can delete own prompts + read=AccessLevel.GROUP, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, )) if viewerId: - # Viewer: MY-level read + GROUP-level read tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="Prompt", + item="data.system.Prompt", view=True, - read=AccessLevel.GROUP, # Can read group prompts + read=AccessLevel.GROUP, create=AccessLevel.NONE, update=AccessLevel.NONE, delete=AccessLevel.NONE, @@ -542,7 +540,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="Invitation", + item="data.system.Invitation", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -553,7 +551,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="Invitation", + item="data.system.Invitation", view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -564,7 +562,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="Invitation", + item="data.system.Invitation", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -578,20 +576,20 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="AuthEvent", + item="data.system.AuthEvent", view=True, - read=AccessLevel.ALL, # Admin can see all auth events for security monitoring - create=AccessLevel.NONE, # Events are system-generated - update=AccessLevel.NONE, # Audit logs are immutable - delete=AccessLevel.NONE, # NO delete - audit integrity! + read=AccessLevel.ALL, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, )) if userId: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="AuthEvent", + item="data.system.AuthEvent", view=True, - read=AccessLevel.MY, # Users can see their own auth events + read=AccessLevel.MY, create=AccessLevel.NONE, update=AccessLevel.NONE, delete=AccessLevel.NONE, @@ -600,7 +598,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="AuthEvent", + item="data.system.AuthEvent", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index b34b2e36..aec97b5a 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -21,6 +21,27 @@ from modules.security.rootAccess import getRootDbAppConnector logger = logging.getLogger(__name__) +def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str: + """ + Build the standardized objectKey for a DATA context item. + + Format: + - System tables: data.system.{TableName} + - Feature tables: data.feature.{featureCode}.{TableName} + + Args: + tableName: The database table name (e.g., "UserInDB", "TrusteePosition") + featureCode: Optional feature code (e.g., "trustee", "realestate") + If None, assumes system table. + + Returns: + Full objectKey string (e.g., "data.system.UserInDB" or "data.feature.trustee.TrusteePosition") + """ + if featureCode: + return f"data.feature.{featureCode}.{tableName}" + return f"data.system.{tableName}" + + def getRecordsetWithRBAC( connector, # DatabaseConnector instance modelClass: Type[BaseModel], @@ -31,6 +52,7 @@ def getRecordsetWithRBAC( mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None, enrichPermissions: bool = False, + featureCode: Optional[str] = None, ) -> List[Dict[str, Any]]: """ Get records with RBAC filtering applied at database level. @@ -50,11 +72,15 @@ def getRecordsetWithRBAC( featureInstanceId: Explicit feature instance context enrichPermissions: If True, adds _permissions field to each record with row-level permissions { canUpdate, canDelete } based on RBAC rules and _createdBy + featureCode: Optional feature code for feature-specific tables (e.g., "trustee"). + If None, table is treated as a system table. Returns: List of filtered records (with _permissions if enrichPermissions=True) """ table = modelClass.__name__ + # Build full objectKey for RBAC lookup + objectKey = buildDataObjectKey(table, featureCode) effectiveMandateId = mandateId @@ -74,21 +100,21 @@ def getRecordsetWithRBAC( record["_permissions"] = {"canUpdate": True, "canDelete": True} return records - # Get RBAC permissions for this table + # Get RBAC permissions for this table using full objectKey # AccessRule table is always in DbApp database dbApp = getRootDbAppConnector() rbacInstance = RbacClass(connector, dbApp=dbApp) permissions = rbacInstance.getUserPermissions( currentUser, AccessRuleContext.DATA, - table, + objectKey, # Use full objectKey (e.g., "data.system.UserInDB") mandateId=effectiveMandateId, featureInstanceId=featureInstanceId ) # Check view permission first if not permissions.view: - logger.debug(f"User {currentUser.id} has no view permission for table {table}") + logger.debug(f"User {currentUser.id} has no view permission for {objectKey}") return [] # Build WHERE clause with RBAC filtering diff --git a/modules/routes/routeAdminRbacExport.py b/modules/routes/routeAdminRbacExport.py index c44c3b6b..2164cb48 100644 --- a/modules/routes/routeAdminRbacExport.py +++ b/modules/routes/routeAdminRbacExport.py @@ -207,7 +207,7 @@ async def import_global_rbac( existingRole = existingRoles[0] roleId = existingRole.get("id") - rootInterface.db.recordUpdate( + rootInterface.db.recordModify( Role, roleId, { @@ -469,7 +469,7 @@ async def import_mandate_rbac( existingRole = existingRoles[0] roleId = existingRole.get("id") - rootInterface.db.recordUpdate( + rootInterface.db.recordModify( Role, roleId, {"description": roleData.get("description", {})} diff --git a/modules/routes/routeAdminUserAccessOverview.py b/modules/routes/routeAdminUserAccessOverview.py new file mode 100644 index 00000000..f12fe2b6 --- /dev/null +++ b/modules/routes/routeAdminUserAccessOverview.py @@ -0,0 +1,503 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Admin User Access Overview routes. +Provides endpoints for viewing complete user access permissions. + +MULTI-TENANT: These are SYSTEM-LEVEL operations requiring isSysAdmin=true. +Shows comprehensive view of what a user can see and access. +""" + +from fastapi import APIRouter, HTTPException, Depends, Query, Path, Request +from typing import List, Dict, Any, Optional, Set +import logging + +from modules.auth import limiter, requireSysAdmin +from modules.datamodels.datamodelUam import User, UserInDB +from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext +from modules.datamodels.datamodelMembership import ( + UserMandate, + UserMandateRole, + FeatureAccess, + FeatureAccessRole, +) +from modules.datamodels.datamodelFeatures import FeatureInstance, Feature +from modules.interfaces.interfaceDbApp import getRootInterface + +# Configure logger +logger = logging.getLogger(__name__) + + +router = APIRouter( + prefix="/api/admin/user-access-overview", + tags=["Admin User Access Overview"], + responses={404: {"description": "Not found"}} +) + + +def _getAccessLevelLabel(level: Optional[str]) -> str: + """Convert access level code to human-readable label.""" + labels = { + "a": "ALL", + "m": "MY", + "g": "GROUP", + "n": "NONE", + None: "-" + } + return labels.get(level, "-") + + +def _getRoleScope(role: Dict[str, Any]) -> str: + """Determine the scope of a role.""" + if role.get("featureInstanceId"): + return "instance" + elif role.get("mandateId"): + return "mandate" + else: + return "global" + + +def _getRoleScopePriority(scope: str) -> int: + """Get priority for role scope (higher = more specific).""" + priorities = {"global": 1, "mandate": 2, "instance": 3} + return priorities.get(scope, 0) + + +@router.get("/users", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def listUsersForOverview( + request: Request, + currentUser: User = Depends(requireSysAdmin) +) -> List[Dict[str, Any]]: + """ + Get list of all users for selection in the overview. + MULTI-TENANT: SysAdmin-only. + + Returns: + - List of user dictionaries with basic info + """ + try: + interface = getRootInterface() + + # Get all users + allUsersData = interface.db.getRecordset(UserInDB) + + result = [] + for u in allUsersData: + result.append({ + "id": u.get("id"), + "username": u.get("username"), + "email": u.get("email"), + "fullName": u.get("fullName"), + "isSysAdmin": u.get("isSysAdmin", False), + "enabled": u.get("enabled", True), + }) + + # Sort by username + result.sort(key=lambda x: (x.get("username") or "").lower()) + + return result + + except Exception as e: + logger.error(f"Error listing users for overview: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to list users: {str(e)}" + ) + + +@router.get("/{userId}", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getUserAccessOverview( + request: Request, + userId: str = Path(..., description="User ID to get access overview for"), + mandateId: Optional[str] = Query(None, description="Filter by mandate ID"), + featureInstanceId: Optional[str] = Query(None, description="Filter by feature instance ID"), + currentUser: User = Depends(requireSysAdmin) +) -> Dict[str, Any]: + """ + Get comprehensive access overview for a specific user. + MULTI-TENANT: SysAdmin-only. + + Path Parameters: + - userId: User ID + + Query Parameters: + - mandateId: Optional filter by mandate ID + - featureInstanceId: Optional filter by feature instance ID + + Returns: + - Comprehensive access overview including: + - User info + - All assigned roles with scope + - UI access (what pages/views the user can see) + - Data access (what tables/fields the user can access) + - Resource access (what resources the user can use) + """ + try: + interface = getRootInterface() + + # Get user + user = interface.getUser(userId) + if not user: + raise HTTPException( + status_code=404, + detail=f"User {userId} not found" + ) + + # Build user info + userInfo = { + "id": user.id, + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "isSysAdmin": user.isSysAdmin, + "enabled": user.enabled, + } + + # If user is SysAdmin, they have full access to everything + if user.isSysAdmin: + return { + "user": userInfo, + "isSysAdmin": True, + "sysAdminNote": "SysAdmin users have full access to all system-level resources without mandate context.", + "roles": [], + "mandates": [], + "uiAccess": [], + "dataAccess": [], + "resourceAccess": [], + } + + # Collect all roles for the user + allRoles = [] + roleIdToInfo = {} # Map roleId to role info for later reference + + # Get mandates for this user + mandateFilter = {"userId": userId, "enabled": True} + if mandateId: + mandateFilter["mandateId"] = mandateId + + userMandates = interface.db.getRecordset(UserMandate, recordFilter=mandateFilter) + + mandatesInfo = [] + for um in userMandates: + umId = um.get("id") + umMandateId = um.get("mandateId") + + # Get mandate name + mandate = interface.getMandate(umMandateId) + mandateName = mandate.name if mandate else umMandateId + + # Get roles for this UserMandate + umRoles = interface.db.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": umId} + ) + + mandateRoleIds = [] + for umr in umRoles: + roleId = umr.get("roleId") + if roleId: + mandateRoleIds.append(roleId) + + # Get role details + roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + scope = _getRoleScope(role) + roleInfo = { + "id": roleId, + "roleLabel": role.get("roleLabel"), + "description": role.get("description", {}), + "scope": scope, + "scopePriority": _getRoleScopePriority(scope), + "mandateId": role.get("mandateId"), + "featureInstanceId": role.get("featureInstanceId"), + "source": "mandate", + "sourceMandateId": umMandateId, + "sourceMandateName": mandateName, + } + allRoles.append(roleInfo) + roleIdToInfo[roleId] = roleInfo + + # Get feature instances for this mandate + featureInstanceFilter = {"userId": userId, "enabled": True} + featureAccesses = interface.db.getRecordset(FeatureAccess, recordFilter=featureInstanceFilter) + + featureInstancesInfo = [] + for fa in featureAccesses: + faId = fa.get("id") + faInstanceId = fa.get("featureInstanceId") + + # Check if instance belongs to this mandate + instance = interface.db.getRecordset(FeatureInstance, recordFilter={"id": faInstanceId}) + if not instance: + continue + instance = instance[0] + + if instance.get("mandateId") != umMandateId: + continue + + # Filter by featureInstanceId if specified + if featureInstanceId and faInstanceId != featureInstanceId: + continue + + # Get feature info + featureCode = instance.get("featureCode") + featureRecords = interface.db.getRecordset(Feature, recordFilter={"code": featureCode}) + featureLabel = featureRecords[0].get("label", {}) if featureRecords else {} + + # Get roles for this FeatureAccess + faRoles = interface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": faId} + ) + + instanceRoleIds = [] + for far in faRoles: + roleId = far.get("roleId") + if roleId: + instanceRoleIds.append(roleId) + + # Get role details (if not already added) + if roleId not in roleIdToInfo: + roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + role = roleRecords[0] + scope = _getRoleScope(role) + roleInfo = { + "id": roleId, + "roleLabel": role.get("roleLabel"), + "description": role.get("description", {}), + "scope": scope, + "scopePriority": _getRoleScopePriority(scope), + "mandateId": role.get("mandateId"), + "featureInstanceId": role.get("featureInstanceId"), + "source": "featureInstance", + "sourceInstanceId": faInstanceId, + "sourceInstanceLabel": instance.get("label"), + } + allRoles.append(roleInfo) + roleIdToInfo[roleId] = roleInfo + + featureInstancesInfo.append({ + "id": faInstanceId, + "label": instance.get("label"), + "featureCode": featureCode, + "featureLabel": featureLabel, + "roleIds": instanceRoleIds, + }) + + mandatesInfo.append({ + "id": umMandateId, + "name": mandateName, + "roleIds": mandateRoleIds, + "featureInstances": featureInstancesInfo, + }) + + # Remove duplicate roles (keep most specific) + uniqueRoles = {} + for role in allRoles: + roleId = role["id"] + if roleId not in uniqueRoles or role["scopePriority"] > uniqueRoles[roleId]["scopePriority"]: + uniqueRoles[roleId] = role + + allRoles = list(uniqueRoles.values()) + + # Get all AccessRules for all role IDs + allRoleIds = list(roleIdToInfo.keys()) + + # Collect access by context + uiAccess = [] + dataAccess = [] + resourceAccess = [] + + for roleId in allRoleIds: + roleInfo = roleIdToInfo.get(roleId, {}) + roleLabel = roleInfo.get("roleLabel", "unknown") + roleScope = roleInfo.get("scope", "unknown") + + # Get all rules for this role + rules = interface.db.getRecordset(AccessRule, recordFilter={"roleId": roleId}) + + for rule in rules: + context = rule.get("context") + item = rule.get("item") + + accessEntry = { + "item": item or "(all)", + "grantedByRoleId": roleId, + "grantedByRoleLabel": roleLabel, + "roleScope": roleScope, + "scopePriority": roleInfo.get("scopePriority", 0), + } + + if context == "UI": + accessEntry["view"] = rule.get("view", False) + if accessEntry["view"]: + uiAccess.append(accessEntry) + + elif context == "DATA": + accessEntry["view"] = rule.get("view", False) + accessEntry["read"] = _getAccessLevelLabel(rule.get("read")) + accessEntry["create"] = _getAccessLevelLabel(rule.get("create")) + accessEntry["update"] = _getAccessLevelLabel(rule.get("update")) + accessEntry["delete"] = _getAccessLevelLabel(rule.get("delete")) + dataAccess.append(accessEntry) + + elif context == "RESOURCE": + accessEntry["view"] = rule.get("view", False) + if accessEntry["view"]: + resourceAccess.append(accessEntry) + + # Merge and deduplicate access entries (keep highest priority) + def _mergeAccessEntries(entries: List[Dict], isDataContext: bool = False) -> List[Dict]: + """Merge entries for same item, keeping highest priority.""" + merged = {} + for entry in entries: + item = entry["item"] + priority = entry.get("scopePriority", 0) + + if item not in merged or priority > merged[item].get("scopePriority", 0): + merged[item] = entry + elif item in merged and priority == merged[item].get("scopePriority", 0): + # Same priority - merge grantedBy info + existingRoles = merged[item].get("grantedByRoleLabels", [merged[item].get("grantedByRoleLabel")]) + if entry["grantedByRoleLabel"] not in existingRoles: + existingRoles.append(entry["grantedByRoleLabel"]) + merged[item]["grantedByRoleLabels"] = existingRoles + + # For DATA context, merge to most permissive + if isDataContext: + levelOrder = {"NONE": 0, "-": 0, "MY": 1, "GROUP": 2, "ALL": 3} + for field in ["read", "create", "update", "delete"]: + existingLevel = merged[item].get(field, "-") + newLevel = entry.get(field, "-") + if levelOrder.get(newLevel, 0) > levelOrder.get(existingLevel, 0): + merged[item][field] = newLevel + + # Clean up and sort + result = list(merged.values()) + for entry in result: + if "grantedByRoleLabels" not in entry: + entry["grantedByRoleLabels"] = [entry.get("grantedByRoleLabel")] + # Remove internal priority field from response + entry.pop("scopePriority", None) + + result.sort(key=lambda x: x.get("item", "")) + return result + + uiAccess = _mergeAccessEntries(uiAccess) + dataAccess = _mergeAccessEntries(dataAccess, isDataContext=True) + resourceAccess = _mergeAccessEntries(resourceAccess) + + # Clean up roles for response + for role in allRoles: + role.pop("scopePriority", None) + + # Sort roles by scope (instance > mandate > global) then by label + allRoles.sort(key=lambda r: (-_getRoleScopePriority(r.get("scope", "")), r.get("roleLabel", "").lower())) + + return { + "user": userInfo, + "isSysAdmin": False, + "roles": allRoles, + "mandates": mandatesInfo, + "uiAccess": uiAccess, + "dataAccess": dataAccess, + "resourceAccess": resourceAccess, + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting user access overview: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to get user access overview: {str(e)}" + ) + + +@router.get("/{userId}/effective-permissions", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def getEffectivePermissions( + request: Request, + userId: str = Path(..., description="User ID"), + mandateId: str = Query(..., description="Mandate ID context"), + featureInstanceId: Optional[str] = Query(None, description="Feature instance ID context"), + context: str = Query("DATA", description="Context type: DATA, UI, or RESOURCE"), + item: Optional[str] = Query(None, description="Specific item to check permissions for"), + currentUser: User = Depends(requireSysAdmin) +) -> Dict[str, Any]: + """ + Get effective (resolved) permissions for a user in a specific context. + This uses the RBAC resolution logic to show what permissions actually apply. + MULTI-TENANT: SysAdmin-only. + + Path Parameters: + - userId: User ID + + Query Parameters: + - mandateId: Required mandate context + - featureInstanceId: Optional feature instance context + - context: Permission context (DATA, UI, RESOURCE) + - item: Optional specific item to check + + Returns: + - Effective permissions after RBAC resolution + """ + try: + interface = getRootInterface() + + # Get user + user = interface.getUser(userId) + if not user: + raise HTTPException( + status_code=404, + detail=f"User {userId} not found" + ) + + # Convert context string to enum + try: + contextEnum = AccessRuleContext(context) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Invalid context: {context}. Must be DATA, UI, or RESOURCE." + ) + + # Use RBAC interface to get actual permissions + from modules.security.rbac import RbacClass + rbac = RbacClass(interface.db, dbApp=interface.db) + + permissions = rbac.getUserPermissions( + user=user, + context=contextEnum, + item=item or "", + mandateId=mandateId, + featureInstanceId=featureInstanceId + ) + + return { + "userId": userId, + "mandateId": mandateId, + "featureInstanceId": featureInstanceId, + "context": context, + "item": item, + "effectivePermissions": { + "view": permissions.view, + "read": _getAccessLevelLabel(permissions.read.value if permissions.read else None), + "create": _getAccessLevelLabel(permissions.create.value if permissions.create else None), + "update": _getAccessLevelLabel(permissions.update.value if permissions.update else None), + "delete": _getAccessLevelLabel(permissions.delete.value if permissions.delete else None), + } + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting effective permissions: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Failed to get effective permissions: {str(e)}" + ) diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index c0b219ec..3f06810f 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -20,10 +20,11 @@ import json from pydantic import BaseModel, Field from modules.auth import limiter, getCurrentUser -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, UserInDB, Mandate, UserConnection from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp from modules.shared.auditLogger import audit_logger +from modules.shared.gdprDeletion import deleteUserDataAcrossAllDatabases, buildDeletionSummary logger = logging.getLogger(__name__) @@ -98,8 +99,7 @@ async def export_user_data( "id": str(currentUser.id), "username": currentUser.username, "email": currentUser.email, - "firstname": currentUser.firstname, - "lastname": currentUser.lastname, + "fullName": getattr(currentUser, "fullName", None), "enabled": currentUser.enabled, "isSysAdmin": getattr(currentUser, "isSysAdmin", False), "createdAt": getattr(currentUser, "createdAt", None), @@ -257,17 +257,11 @@ async def export_portable_data( "@context": "https://schema.org", "@type": "Person", "identifier": str(currentUser.id), - "name": f"{currentUser.firstname or ''} {currentUser.lastname or ''}".strip() or currentUser.username, + "name": getattr(currentUser, "fullName", None) or currentUser.username, "email": currentUser.email, "additionalProperty": [] } - # Add profile properties - if currentUser.firstname: - portableData["givenName"] = currentUser.firstname - if currentUser.lastname: - portableData["familyName"] = currentUser.lastname - # Add mandate memberships as organization affiliations from modules.datamodels.datamodelMembership import UserMandate userMandates = rootInterface.db.getRecordset( @@ -364,10 +358,17 @@ async def delete_account( ) try: - rootInterface = getRootInterface() - deletedData = [] + # Step 1: Audit log BEFORE deletion (audit needs userId) + audit_logger.logGdprEvent( + userId=str(currentUser.id), + mandateId="system", + action="gdpr_account_deletion_started", + details=f"User initiated account deletion (GDPR Article 17 - Right to Erasure)", + ipAddress=request.client.host if request.client else None + ) - # 1. Revoke all invitations created by user + # Step 2: Revoke invitations BEFORE generic deletion (business logic) + rootInterface = getRootInterface() from modules.datamodels.datamodelInvitation import Invitation userInvitations = rootInterface.db.getRecordset( Invitation, @@ -375,78 +376,37 @@ async def delete_account( ) for inv in userInvitations: - rootInterface.db.recordUpdate( + rootInterface.db.recordModify( Invitation, inv.get("id"), {"revokedAt": getUtcTimestamp()} ) - deletedData.append(f"Invitations revoked: {len(userInvitations)}") - # 2. Delete feature accesses (CASCADE will delete FeatureAccessRoles) - from modules.datamodels.datamodelMembership import FeatureAccess - featureAccesses = rootInterface.db.getRecordset( - FeatureAccess, - recordFilter={"userId": str(currentUser.id)} - ) + logger.info(f"Revoked {len(userInvitations)} invitations for user {currentUser.id}") - for fa in featureAccesses: - rootInterface.db.recordDelete(FeatureAccess, fa.get("id")) - deletedData.append(f"Feature accesses deleted: {len(featureAccesses)}") + # Step 3: Generic deletion across ALL databases + deletionStats = deleteUserDataAcrossAllDatabases(str(currentUser.id), currentUser) - # 3. Delete mandate memberships (CASCADE will delete UserMandateRoles) - from modules.datamodels.datamodelMembership import UserMandate - userMandates = rootInterface.db.getRecordset( - UserMandate, - recordFilter={"userId": str(currentUser.id)} - ) - - for um in userMandates: - rootInterface.db.recordDelete(UserMandate, um.get("id")) - deletedData.append(f"Mandate memberships deleted: {len(userMandates)}") - - # 4. Delete active tokens - from modules.datamodels.datamodelSecurity import Token - userTokens = rootInterface.db.getRecordset( - Token, - recordFilter={"userId": str(currentUser.id)} - ) - - for token in userTokens: - rootInterface.db.recordDelete(Token, token.get("id")) - deletedData.append(f"Tokens deleted: {len(userTokens)}") - - # 5. Delete user connections (OAuth) - userConnections = rootInterface.db.getRecordset( - UserConnection, - recordFilter={"userId": str(currentUser.id)} - ) - - for conn in userConnections: - rootInterface.db.recordDelete(UserConnection, conn.get("id")) - deletedData.append(f"Connections deleted: {len(userConnections)}") - - # 6. Finally, delete the user + # Step 4: Delete the user account from UserInDB (authentication table) + # This must be done AFTER all other deletions to maintain audit trail deletedAt = getUtcTimestamp() - rootInterface.db.recordDelete(User, str(currentUser.id)) - deletedData.append("User account deleted") + rootInterface.db.recordDelete(UserInDB, str(currentUser.id)) - # Audit log (before user is deleted) - GDPR Article 17 account deletion - audit_logger.logGdprEvent( - userId=str(currentUser.id), - mandateId="system", - action="gdpr_account_deletion", - details=f"User deleted own account (GDPR Article 17 - Right to Erasure). Data: {', '.join(deletedData)}", - ipAddress=request.client.host if request.client else None - ) + # Build summary for response + deletedData = buildDeletionSummary(deletionStats) + deletedData.insert(0, f"Invitations revoked: {len(userInvitations)}") + deletedData.append("User account deleted from authentication system") - logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17)") + logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17). " + f"Stats: {deletionStats['totalRecordsDeleted']} deleted, " + f"{deletionStats['totalRecordsAnonymized']} anonymized") return DeletionResult( success=True, userId=str(currentUser.id), deletedAt=deletedAt, deletedData=deletedData, - message="Account and all associated data have been permanently deleted." + message="Account and all associated data have been permanently deleted or anonymized." ) except HTTPException: diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index 0e0259eb..47fda648 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -82,26 +82,6 @@ class InvitationValidation(BaseModel): roleIds: List[str] -class RegisterAndAcceptRequest(BaseModel): - """Request model for combined registration + invitation acceptance""" - token: str = Field(..., description="Invitation token") - username: str = Field(..., min_length=3, max_length=50, description="Username for the new account") - email: str = Field(..., description="Email address") - password: str = Field(..., min_length=8, description="Password (min 8 characters)") - firstname: Optional[str] = Field(None, description="First name") - lastname: Optional[str] = Field(None, description="Last name") - - -class RegisterAndAcceptResponse(BaseModel): - """Response model for combined registration + invitation acceptance""" - message: str - userId: str - mandateId: str - userMandateId: str - featureAccessId: Optional[str] - roleIds: List[str] - - # ============================================================================= # Invitation CRUD Endpoints # ============================================================================= @@ -371,7 +351,7 @@ async def revoke_invitation( ) # Revoke invitation - rootInterface.db.recordUpdate( + rootInterface.db.recordModify( Invitation, invitationId, {"revokedAt": getUtcTimestamp()} @@ -575,7 +555,7 @@ async def accept_invitation( featureAccessId = str(featureAccess.id) # Update invitation usage - rootInterface.db.recordUpdate( + rootInterface.db.recordModify( Invitation, invitation.get("id"), { @@ -608,161 +588,6 @@ async def accept_invitation( ) -# ============================================================================= -# Combined Registration + Accept Invitation -# ============================================================================= - -@router.post("/register-and-accept", response_model=RegisterAndAcceptResponse) -@limiter.limit("10/minute") # Stricter rate limit for registration -async def register_and_accept_invitation( - request: Request, - data: RegisterAndAcceptRequest -) -> RegisterAndAcceptResponse: - """ - Combined endpoint: Register a new user AND accept an invitation in one step. - - This is a PUBLIC endpoint - no authentication required. - - Flow: - 1. Validate invitation token - 2. Check email matches (if invitation has email restriction) - 3. Create new user account - 4. Create UserMandate membership with roles - 5. Optionally grant FeatureAccess - 6. Update invitation usage - - The user can then login with their new credentials. - """ - try: - rootInterface = getRootInterface() - - # 1. Validate invitation - invitation = rootInterface.getInvitationByToken(data.token) - if not invitation: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Invalid invitation token" - ) - - if invitation.get("revokedAt"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invitation has been revoked" - ) - - currentTime = getUtcTimestamp() - if invitation.get("expiresAt", 0) < currentTime: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invitation has expired" - ) - - if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invitation has reached maximum uses" - ) - - # 2. Check email restriction - invitationEmail = invitation.get("email") - if invitationEmail and invitationEmail.lower() != data.email.lower(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Email does not match the invitation" - ) - - # 3. Check if username or email already exists - existingUsername = rootInterface.getUserByUsername(data.username) - if existingUsername: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Username already exists" - ) - - existingEmail = rootInterface.getUserByEmail(data.email) - if existingEmail: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="Email already registered. Please login and accept the invitation." - ) - - # 4. Create new user - from modules.security.passwordUtils import hashPassword - hashedPassword = hashPassword(data.password) - - newUser = rootInterface.createUser( - username=data.username, - email=data.email, - passwordHash=hashedPassword, - firstname=data.firstname, - lastname=data.lastname - ) - - if not newUser: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Failed to create user account" - ) - - userId = str(newUser.id) - mandateId = invitation.get("mandateId") - roleIds = invitation.get("roleIds", []) - featureInstanceId = invitation.get("featureInstanceId") - - # 5. Create UserMandate membership - userMandate = rootInterface.createUserMandate( - userId=userId, - mandateId=mandateId, - roleIds=roleIds - ) - userMandateId = str(userMandate.id) - - # 6. Grant feature access if specified - featureAccessId = None - if featureInstanceId: - instanceRoleIds = [r for r in roleIds if _isInstanceRole(rootInterface, r, featureInstanceId)] - featureAccess = rootInterface.createFeatureAccess( - userId=userId, - featureInstanceId=featureInstanceId, - roleIds=instanceRoleIds - ) - featureAccessId = str(featureAccess.id) - - # 7. Update invitation usage - rootInterface.db.recordUpdate( - Invitation, - invitation.get("id"), - { - "currentUses": invitation.get("currentUses", 0) + 1, - "usedBy": userId, - "usedAt": currentTime - } - ) - - logger.info( - f"New user {userId} registered and accepted invitation {invitation.get('id')} " - f"for mandate {mandateId}" - ) - - return RegisterAndAcceptResponse( - message="Account created and invitation accepted successfully", - userId=userId, - mandateId=mandateId, - userMandateId=userMandateId, - featureAccessId=featureAccessId, - roleIds=roleIds - ) - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error in register-and-accept: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to complete registration: {str(e)}" - ) - - # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeSharepoint.py b/modules/routes/routeSharepoint.py index aa62afc6..32c72597 100644 --- a/modules/routes/routeSharepoint.py +++ b/modules/routes/routeSharepoint.py @@ -146,3 +146,108 @@ async def list_sharepoint_folders( detail=f"Error listing SharePoint folders: {str(e)}" ) + +@router.get("/{connectionId}/folder-options", response_model=List[Dict[str, Any]]) +@limiter.limit("30/minute") +async def getSharepointFolderOptions( + request: Request, + connectionId: str = Path(..., description="Microsoft connection ID"), + siteId: Optional[str] = Query(None, description="Specific site ID to browse (if omitted, returns sites only)"), + path: Optional[str] = Query(None, description="Folder path within site to browse"), + currentUser: User = Depends(getCurrentUser) +) -> List[Dict[str, Any]]: + """ + Get SharePoint folders formatted as dropdown options. + + Two modes: + 1. If siteId is not provided: Returns list of sites (for site selection) + 2. If siteId is provided: Returns folders within that site (optionally at specific path) + + This avoids expensive iteration through all sites and folders. + """ + try: + interface = getInterface(currentUser) + + # Get the connection and verify it belongs to the user + connection = _getUserConnection(interface, connectionId, currentUser.id) + if not connection: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Connection {connectionId} not found or does not belong to user" + ) + + # Verify it's a Microsoft connection + authority = connection.authority.value if hasattr(connection.authority, 'value') else str(connection.authority) + if authority.lower() != 'msft': + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Connection {connectionId} is not a Microsoft connection" + ) + + # Initialize services + services = getServices(currentUser, None) + + # Set access token on SharePoint service + if not services.sharepoint.setAccessTokenFromConnection(connection): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Failed to set SharePoint access token. Connection may be expired or invalid." + ) + + # Mode 1: Return sites list if no siteId specified + if not siteId: + sites = await services.sharepoint.discoverSites() + return [ + { + "type": "site", + "value": site.get("id"), + "label": site.get("displayName", "Unknown Site"), + "siteId": site.get("id"), + "siteName": site.get("displayName", "Unknown Site"), + "webUrl": site.get("webUrl", ""), + "path": _extractSitePath(site.get("webUrl", "")) + } + for site in sites + ] + + # Mode 2: Return folders within specific site + folderPath = path or "" + items = await services.sharepoint.listFolderContents(siteId, folderPath) + + if not items: + return [] + + folderOptions = [] + for item in items: + if item.get("type") == "folder": + folderName = item.get("name", "") + itemPath = f"{folderPath}/{folderName}" if folderPath else folderName + + folderOptions.append({ + "type": "folder", + "value": itemPath, + "label": folderName, + "siteId": siteId, + "folderName": folderName, + "path": itemPath, + "hasChildren": True # Assume folders may have children + }) + + return folderOptions + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting SharePoint folder options: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error getting SharePoint folder options: {str(e)}" + ) + + +def _extractSitePath(webUrl: str) -> str: + """Extract site path from webUrl (e.g., https://company.sharepoint.com/sites/MySite -> /sites/MySite)""" + if "/sites/" in webUrl: + return "/sites/" + webUrl.split("/sites/")[1].split("/")[0] + return "" + diff --git a/modules/system/routeSystem.py b/modules/routes/routeSystem.py similarity index 100% rename from modules/system/routeSystem.py rename to modules/routes/routeSystem.py diff --git a/modules/security/rbac.py b/modules/security/rbac.py index d35120cb..34e80105 100644 --- a/modules/security/rbac.py +++ b/modules/security/rbac.py @@ -381,10 +381,20 @@ class RbacClass: """ Check if a rule matches the given item. + Matching rules (in order of specificity): + 1. Generic rule (item=None) matches everything + 2. Exact match (rule.item == item) + 3. Prefix match (item starts with rule.item + ".") + Example: rule "data.feature.trustee" matches item "data.feature.trustee.TrusteePosition" + + All items MUST use the full objectKey format: + - System: data.system.{TableName} (e.g., "data.system.UserInDB") + - Feature: data.feature.{featureCode}.{TableName} (e.g., "data.feature.trustee.TrusteePosition") + - UI: ui.{area}.{page} (e.g., "ui.admin.users") + Args: rule: Access rule to check - item: Item to match against (can be short name like "TrusteePosition" or - fully qualified like "data.feature.trustee.TrusteePosition") + item: Full objectKey to match against Returns: True if rule matches item @@ -401,15 +411,10 @@ class RbacClass: if rule.item == item: return True - # Prefix match (e.g., "trustee" matches "trustee.contract") + # Prefix match (e.g., "data.feature.trustee" matches "data.feature.trustee.TrusteePosition") if item.startswith(rule.item + "."): return True - # Suffix match: rule.item ends with ".{item}" (e.g., "data.feature.trustee.TrusteePosition" matches "TrusteePosition") - # This allows short table names to match fully qualified objectKeys - if rule.item.endswith("." + item): - return True - return False def findMostSpecificRule(self, rules: List[AccessRule], item: str) -> Optional[AccessRule]: diff --git a/modules/shared/gdprDeletion.py b/modules/shared/gdprDeletion.py new file mode 100644 index 00000000..da8a60cf --- /dev/null +++ b/modules/shared/gdprDeletion.py @@ -0,0 +1,679 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Generic GDPR data deletion engine. +Automatically discovers and deletes user data across all databases and tables. + +Design: +- Schema-based: Inspects database schemas to find user-related columns +- Generic: Works with any new features/tables without code changes +- Safe: Anonymizes audit logs instead of deleting them +- Comprehensive: Covers all databases (App, Management, Chat, Feature-DBs) +""" + +import logging +from typing import List, Dict, Any, Set, Tuple +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +# Tables to SKIP (never delete from these) +PROTECTED_TABLES = { + "_system", # System metadata table + "UserInDB", # User account table (deleted separately at the end) +} + +# Tables to ANONYMIZE instead of DELETE (for compliance) +ANONYMIZE_TABLES = { + "AuditEvent", # Audit logs must be retained for compliance + "AuthEvent", # Authentication logs must be retained for compliance +} + +# User reference column patterns to search for +USER_COLUMNS = [ + "userId", + "createdBy", + "usedBy", + "revokedBy", + "_createdBy", + "_modifiedBy", +] + + +def _getTableColumns(dbConnector, tableName: str) -> List[str]: + """ + Get all column names for a table by inspecting the schema. + + Args: + dbConnector: DatabaseConnector instance + tableName: Name of the table + + Returns: + List of column names + """ + try: + query = """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = %s + AND table_schema = 'public' + ORDER BY ordinal_position + """ + + cursor = dbConnector.connection.cursor() + cursor.execute(query, (tableName,)) + columns = [row[0] for row in cursor.fetchall()] + cursor.close() + + return columns + except Exception as e: + logger.error(f"Error getting columns for table {tableName}: {e}") + return [] + + +def _getAllTables(dbConnector) -> List[str]: + """ + Get all table names from a database, sorted by dependency order. + Child tables (with foreign keys) come before parent tables. + + Args: + dbConnector: DatabaseConnector instance + + Returns: + List of table names in deletion order + """ + try: + # Get all tables + query = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """ + + cursor = dbConnector.connection.cursor() + cursor.execute(query) + allTables = [row[0] for row in cursor.fetchall()] + + # Get foreign key relationships to determine dependency order + fkQuery = """ + SELECT + tc.table_name, + ccu.table_name AS foreign_table_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = 'public' + """ + + cursor.execute(fkQuery) + foreignKeys = cursor.fetchall() + cursor.close() + + # Build dependency graph (child -> parent mapping) + dependencies = {} + for childTable, parentTable in foreignKeys: + if childTable not in dependencies: + dependencies[childTable] = [] + dependencies[childTable].append(parentTable) + + # Sort tables by dependency (topological sort) + sortedTables = [] + visited = set() + + def visit(table): + if table in visited or table not in allTables: + return + visited.add(table) + + # Visit dependencies first (parents) + if table in dependencies: + for parent in dependencies[table]: + visit(parent) + + sortedTables.append(table) + + # Visit all tables + for table in allTables: + visit(table) + + # Reverse to get deletion order (children before parents) + sortedTables.reverse() + + # Filter out protected tables + return [t for t in sortedTables if t not in PROTECTED_TABLES] + + except Exception as e: + logger.error(f"Error getting tables from database: {e}") + # Fallback: return simple list without ordering + try: + query = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'" + cursor = dbConnector.connection.cursor() + cursor.execute(query) + tables = [row[0] for row in cursor.fetchall()] + cursor.close() + return [t for t in tables if t not in PROTECTED_TABLES] + except Exception: + return [] + + +def _getPrimaryKeyColumns(dbConnector, tableName: str) -> List[str]: + """ + Get primary key column(s) for a table. + + Args: + dbConnector: DatabaseConnector instance + tableName: Name of the table + + Returns: + List of primary key column names + """ + try: + query = """ + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid + AND a.attnum = ANY(i.indkey) + WHERE i.indrelid = %s::regclass + AND i.indisprimary + """ + + cursor = dbConnector.connection.cursor() + cursor.execute(query, (tableName,)) + pkColumns = [row[0] for row in cursor.fetchall()] + cursor.close() + + return pkColumns + except Exception as e: + logger.debug(f"Could not get primary key for {tableName}: {e}") + return ["id"] # Fallback to 'id' + + +def _findUserReferencesInTable( + dbConnector, + tableName: str, + userId: str +) -> Dict[str, List[Tuple]]: + """ + Find all records in a table that reference a user. + + Args: + dbConnector: DatabaseConnector instance + tableName: Name of the table + userId: User ID to search for + + Returns: + Dict mapping column names to lists of primary key tuples + """ + try: + # Get all columns for this table + columns = _getTableColumns(dbConnector, tableName) + + # Find user-related columns in this table + userColumns = [col for col in columns if col in USER_COLUMNS] + + if not userColumns: + return {} + + # Get primary key columns + pkColumns = _getPrimaryKeyColumns(dbConnector, tableName) + + if not pkColumns: + logger.warning(f"Table {tableName} has no primary key, skipping") + return {} + + references = {} + cursor = dbConnector.connection.cursor() + + for userColumn in userColumns: + # Build SELECT for primary key columns + pkSelect = ", ".join([f'"{pk}"' for pk in pkColumns]) + query = f'SELECT {pkSelect} FROM "{tableName}" WHERE "{userColumn}" = %s' + + cursor.execute(query, (userId,)) + recordKeys = cursor.fetchall() + + if recordKeys: + references[userColumn] = recordKeys + logger.debug(f"Found {len(recordKeys)} records in {tableName}.{userColumn} for user {userId}") + + cursor.close() + return references + + except Exception as e: + logger.error(f"Error finding user references in {tableName}: {e}") + return {} + + +def _anonymizeRecords( + dbConnector, + tableName: str, + columnName: str, + recordKeys: List[Tuple], + pkColumns: List[str], + anonymousValue: str = "deleted_user" +) -> int: + """ + Anonymize user references in records (set to 'deleted_user'). + + Args: + dbConnector: DatabaseConnector instance + tableName: Name of the table + columnName: Name of the column to anonymize + recordKeys: List of primary key tuples + pkColumns: List of primary key column names + anonymousValue: Value to set (default: "deleted_user") + + Returns: + Number of records anonymized + """ + if not recordKeys: + return 0 + + try: + cursor = dbConnector.connection.cursor() + count = 0 + + for recordKey in recordKeys: + # Build WHERE clause for primary key + whereClause = " AND ".join([f'"{pk}" = %s' for pk in pkColumns]) + + # Check if table has _modifiedAt column + columns = _getTableColumns(dbConnector, tableName) + hasModifiedAt = "_modifiedAt" in columns + + if hasModifiedAt: + query = f'UPDATE "{tableName}" SET "{columnName}" = %s, "_modifiedAt" = %s WHERE {whereClause}' + params = [anonymousValue, getUtcTimestamp()] + else: + query = f'UPDATE "{tableName}" SET "{columnName}" = %s WHERE {whereClause}' + params = [anonymousValue] + + # Add primary key values to params + if isinstance(recordKey, tuple): + params.extend(recordKey) + else: + params.append(recordKey) + + cursor.execute(query, params) + count += cursor.rowcount + + dbConnector.connection.commit() + cursor.close() + + logger.info(f"Anonymized {count} records in {tableName}.{columnName}") + return count + + except Exception as e: + logger.error(f"Error anonymizing records in {tableName}.{columnName}: {e}") + dbConnector.connection.rollback() + return 0 + + +def _deleteRecords( + dbConnector, + tableName: str, + recordKeys: List[Tuple], + pkColumns: List[str] +) -> int: + """ + Delete records from a table. + + Args: + dbConnector: DatabaseConnector instance + tableName: Name of the table + recordKeys: List of primary key tuples + pkColumns: List of primary key column names + + Returns: + Number of records deleted + """ + if not recordKeys: + return 0 + + try: + cursor = dbConnector.connection.cursor() + count = 0 + + for recordKey in recordKeys: + # Build WHERE clause for primary key + whereClause = " AND ".join([f'"{pk}" = %s' for pk in pkColumns]) + query = f'DELETE FROM "{tableName}" WHERE {whereClause}' + + # Prepare params + if isinstance(recordKey, tuple): + params = list(recordKey) + else: + params = [recordKey] + + cursor.execute(query, params) + count += cursor.rowcount + + dbConnector.connection.commit() + cursor.close() + + logger.info(f"Deleted {count} records from {tableName}") + return count + + except Exception as e: + logger.error(f"Error deleting records from {tableName}: {e}") + dbConnector.connection.rollback() + return 0 + + +def deleteUserDataFromDatabase( + dbConnector, + userId: str, + databaseName: str +) -> Dict[str, Any]: + """ + Delete or anonymize all user data from a single database. + + Args: + dbConnector: DatabaseConnector instance + userId: User ID to delete + databaseName: Name of the database (for logging) + + Returns: + Dict with deletion statistics + """ + stats = { + "database": databaseName, + "tablesProcessed": 0, + "recordsDeleted": 0, + "recordsAnonymized": 0, + "errors": [] + } + + try: + # Get all tables in this database + tables = _getAllTables(dbConnector) + + logger.info(f"Processing {len(tables)} tables in {databaseName} for user {userId}") + + for tableName in tables: + try: + # Get primary key columns for this table + pkColumns = _getPrimaryKeyColumns(dbConnector, tableName) + + if not pkColumns: + logger.debug(f"Skipping {tableName} - no primary key") + continue + + # Find user references in this table + references = _findUserReferencesInTable(dbConnector, tableName, userId) + + if not references: + continue + + stats["tablesProcessed"] += 1 + + # Decide: Anonymize or Delete? + shouldAnonymize = tableName in ANONYMIZE_TABLES + + for columnName, recordKeys in references.items(): + if shouldAnonymize: + # Anonymize audit/log tables + count = _anonymizeRecords( + dbConnector, tableName, columnName, recordKeys, pkColumns + ) + stats["recordsAnonymized"] += count + else: + # Delete from regular tables + count = _deleteRecords(dbConnector, tableName, recordKeys, pkColumns) + stats["recordsDeleted"] += count + + except Exception as tableErr: + errorMsg = f"Error processing table {tableName}: {tableErr}" + logger.error(errorMsg) + stats["errors"].append(errorMsg) + + logger.info(f"Completed deletion in {databaseName}: {stats}") + return stats + + except Exception as e: + errorMsg = f"Error processing database {databaseName}: {e}" + logger.error(errorMsg) + stats["errors"].append(errorMsg) + return stats + + +def deleteUserDataAcrossAllDatabases(userId: str, currentUser) -> Dict[str, Any]: + """ + Delete or anonymize all user data across ALL databases. + + This is the main entry point for GDPR Article 17 (Right to Erasure). + + Features: + - Automatically discovers all databases and tables + - Schema-based: No hardcoded table lists + - Safe: Anonymizes audit logs instead of deleting them + - Comprehensive: Covers App, Management, Chat, and all Feature DBs + + Args: + userId: User ID to delete + currentUser: User object (for interface access) + + Returns: + Dict with comprehensive deletion statistics + """ + allStats = { + "userId": userId, + "deletedAt": getUtcTimestamp(), + "databases": [], + "totalTablesProcessed": 0, + "totalRecordsDeleted": 0, + "totalRecordsAnonymized": 0, + "errors": [] + } + + try: + # Import all database interfaces + from modules.interfaces.interfaceDbApp import getRootInterface as getAppInterface + from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + + # 1. Process App DB (poweron_app) + try: + appInterface = getAppInterface() + appStats = deleteUserDataFromDatabase(appInterface.db, userId, "poweron_app") + allStats["databases"].append(appStats) + allStats["totalTablesProcessed"] += appStats["tablesProcessed"] + allStats["totalRecordsDeleted"] += appStats["recordsDeleted"] + allStats["totalRecordsAnonymized"] += appStats["recordsAnonymized"] + allStats["errors"].extend(appStats["errors"]) + except Exception as appErr: + errorMsg = f"Error processing App DB: {appErr}" + logger.error(errorMsg) + allStats["errors"].append(errorMsg) + + # 2. Process Management DB (poweron_management) + try: + mgmtInterface = getMgmtInterface(currentUser) + mgmtStats = deleteUserDataFromDatabase(mgmtInterface.db, userId, "poweron_management") + allStats["databases"].append(mgmtStats) + allStats["totalTablesProcessed"] += mgmtStats["tablesProcessed"] + allStats["totalRecordsDeleted"] += mgmtStats["recordsDeleted"] + allStats["totalRecordsAnonymized"] += mgmtStats["recordsAnonymized"] + allStats["errors"].extend(mgmtStats["errors"]) + except Exception as mgmtErr: + errorMsg = f"Error processing Management DB: {mgmtErr}" + logger.error(errorMsg) + allStats["errors"].append(errorMsg) + + # 3. Process Chat DB (poweron_chat) + try: + chatInterface = getChatInterface(currentUser) + chatStats = deleteUserDataFromDatabase(chatInterface.db, userId, "poweron_chat") + allStats["databases"].append(chatStats) + allStats["totalTablesProcessed"] += chatStats["tablesProcessed"] + allStats["totalRecordsDeleted"] += chatStats["recordsDeleted"] + allStats["totalRecordsAnonymized"] += chatStats["recordsAnonymized"] + allStats["errors"].extend(chatStats["errors"]) + except Exception as chatErr: + errorMsg = f"Error processing Chat DB: {chatErr}" + logger.error(errorMsg) + allStats["errors"].append(errorMsg) + + # 4. Process Feature DBs (discover dynamically) + try: + featureStats = _deleteUserDataFromFeatureDatabases(userId, currentUser) + allStats["databases"].extend(featureStats["databases"]) + allStats["totalTablesProcessed"] += featureStats["totalTablesProcessed"] + allStats["totalRecordsDeleted"] += featureStats["totalRecordsDeleted"] + allStats["totalRecordsAnonymized"] += featureStats["totalRecordsAnonymized"] + allStats["errors"].extend(featureStats["errors"]) + except Exception as featureErr: + errorMsg = f"Error processing Feature DBs: {featureErr}" + logger.error(errorMsg) + allStats["errors"].append(errorMsg) + + # Log summary + logger.info(f"GDPR deletion completed for user {userId}: " + f"{allStats['totalRecordsDeleted']} deleted, " + f"{allStats['totalRecordsAnonymized']} anonymized across " + f"{len(allStats['databases'])} databases") + + return allStats + + except Exception as e: + logger.error(f"Fatal error in deleteUserDataAcrossAllDatabases: {e}") + allStats["errors"].append(f"Fatal error: {e}") + return allStats + + +def _deleteUserDataFromFeatureDatabases(userId: str, currentUser) -> Dict[str, Any]: + """ + Delete user data from all feature-specific databases. + Discovers feature interfaces dynamically. + + Args: + userId: User ID to delete + currentUser: User object + + Returns: + Dict with deletion statistics + """ + stats = { + "databases": [], + "totalTablesProcessed": 0, + "totalRecordsDeleted": 0, + "totalRecordsAnonymized": 0, + "errors": [] + } + + try: + # Get all feature instances for this user to determine which feature DBs to check + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelMembership import FeatureAccess + from modules.datamodels.datamodelFeatures import FeatureInstance + + rootInterface = getRootInterface() + + # Get all feature accesses for this user + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": str(userId)} + ) + + # Collect unique feature codes + featureCodes: Set[str] = set() + for fa in featureAccesses: + instanceId = fa.get("featureInstanceId") + instanceRecords = rootInterface.db.getRecordset( + FeatureInstance, + recordFilter={"id": instanceId} + ) + if instanceRecords: + featureCode = instanceRecords[0].get("featureCode") + if featureCode: + featureCodes.add(featureCode) + + logger.info(f"Found {len(featureCodes)} feature types to process: {featureCodes}") + + # Process each feature type + for featureCode in featureCodes: + try: + dbName = f"poweron_{featureCode}" + + # Try to get feature interface + featureInterface = None + + if featureCode == "trustee": + from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface + featureInterface = getTrusteeInterface(currentUser) + elif featureCode == "realestate": + from modules.features.realestate.interfaceFeatureRealEstate import getInterface as getRealEstateInterface + featureInterface = getRealEstateInterface(currentUser) + elif featureCode == "chatbot": + from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface + featureInterface = getChatbotInterface(currentUser) + elif featureCode == "neutralization": + from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutralizerInterface + featureInterface = getNeutralizerInterface(currentUser) + else: + logger.warning(f"No interface found for feature code: {featureCode}") + continue + + if featureInterface and hasattr(featureInterface, 'db'): + featureStats = deleteUserDataFromDatabase( + featureInterface.db, + userId, + dbName + ) + stats["databases"].append(featureStats) + stats["totalTablesProcessed"] += featureStats["tablesProcessed"] + stats["totalRecordsDeleted"] += featureStats["recordsDeleted"] + stats["totalRecordsAnonymized"] += featureStats["recordsAnonymized"] + stats["errors"].extend(featureStats["errors"]) + + except Exception as featureErr: + errorMsg = f"Error processing feature {featureCode}: {featureErr}" + logger.warning(errorMsg) + stats["errors"].append(errorMsg) + + return stats + + except Exception as e: + logger.error(f"Error in _deleteUserDataFromFeatureDatabases: {e}") + stats["errors"].append(f"Feature DB error: {e}") + return stats + + +def buildDeletionSummary(stats: Dict[str, Any]) -> List[str]: + """ + Build a human-readable summary of the deletion operation. + + Args: + stats: Statistics dict from deleteUserDataAcrossAllDatabases + + Returns: + List of summary strings + """ + summary = [] + + for dbStats in stats.get("databases", []): + dbName = dbStats.get("database", "unknown") + deleted = dbStats.get("recordsDeleted", 0) + anonymized = dbStats.get("recordsAnonymized", 0) + + if deleted > 0 or anonymized > 0: + parts = [] + if deleted > 0: + parts.append(f"{deleted} deleted") + if anonymized > 0: + parts.append(f"{anonymized} anonymized") + summary.append(f"{dbName}: {', '.join(parts)}") + + # Add totals + totalDeleted = stats.get("totalRecordsDeleted", 0) + totalAnonymized = stats.get("totalRecordsAnonymized", 0) + summary.append(f"Total: {totalDeleted} deleted, {totalAnonymized} anonymized") + + return summary diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index cbb33a52..24d1d410 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -244,6 +244,15 @@ NAVIGATION_SECTIONS = [ "order": 90, "adminOnly": True, }, + { + "id": "admin-user-access-overview", + "objectKey": "ui.admin.user-access-overview", + "label": {"en": "User Access Overview", "de": "Benutzer-Zugriffsübersicht", "fr": "Aperçu des accès utilisateur"}, + "icon": "FaUserShield", + "path": "/admin/user-access-overview", + "order": 100, + "adminOnly": True, + }, ], }, ] @@ -290,16 +299,39 @@ UI_OBJECTS = _buildUiObjectsFromNavigation() # ============================================================================= DATA_OBJECTS = [ + # User/Auth tables { - "objectKey": "data.system.User", + "objectKey": "data.system.UserInDB", "label": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, "meta": {"table": "UserInDB"} }, + { + "objectKey": "data.system.AuthEvent", + "label": {"en": "Auth Event", "de": "Auth-Ereignis", "fr": "Événement d'auth"}, + "meta": {"table": "AuthEvent"} + }, + { + "objectKey": "data.system.UserConnection", + "label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"}, + "meta": {"table": "UserConnection"} + }, + # Mandate/Membership tables { "objectKey": "data.system.Mandate", "label": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, "meta": {"table": "Mandate"} }, + { + "objectKey": "data.system.UserMandate", + "label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, + "meta": {"table": "UserMandate"} + }, + { + "objectKey": "data.system.Invitation", + "label": {"en": "Invitation", "de": "Einladung", "fr": "Invitation"}, + "meta": {"table": "Invitation"} + }, + # RBAC tables { "objectKey": "data.system.Role", "label": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, @@ -310,11 +342,13 @@ DATA_OBJECTS = [ "label": {"en": "Access Rule", "de": "Zugriffsregel", "fr": "Règle d'accès"}, "meta": {"table": "AccessRule"} }, + # Feature tables { - "objectKey": "data.system.UserMandate", - "label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, - "meta": {"table": "UserMandate"} + "objectKey": "data.system.FeatureInstance", + "label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"}, + "meta": {"table": "FeatureInstance"} }, + # Content tables { "objectKey": "data.system.Prompt", "label": {"en": "Prompt", "de": "Prompt", "fr": "Prompt"}, @@ -330,16 +364,6 @@ DATA_OBJECTS = [ "label": {"en": "File", "de": "Datei", "fr": "Fichier"}, "meta": {"table": "FileItem"} }, - { - "objectKey": "data.system.UserConnection", - "label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"}, - "meta": {"table": "UserConnection"} - }, - { - "objectKey": "data.system.FeatureInstance", - "label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"}, - "meta": {"table": "FeatureInstance"} - }, ] # ============================================================================= diff --git a/modules/workflows/methods/methodSharepoint/actions/__init__.py b/modules/workflows/methods/methodSharepoint/actions/__init__.py index 6975f8af..d59d29aa 100644 --- a/modules/workflows/methods/methodSharepoint/actions/__init__.py +++ b/modules/workflows/methods/methodSharepoint/actions/__init__.py @@ -13,6 +13,7 @@ from .findSiteByUrl import findSiteByUrl from .downloadFileByPath import downloadFileByPath from .copyFile import copyFile from .uploadFile import uploadFile +from .getExpensesFromPdf import getExpensesFromPdf __all__ = [ 'findDocumentPath', @@ -24,5 +25,6 @@ __all__ = [ 'downloadFileByPath', 'copyFile', 'uploadFile', + 'getExpensesFromPdf', ] diff --git a/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py new file mode 100644 index 00000000..c2ecb7c9 --- /dev/null +++ b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py @@ -0,0 +1,733 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. + +""" +Action to extract expenses from PDF documents in SharePoint and save to TrusteePosition. + +Process: +1. Read PDF files from SharePoint folder (max 50 files per execution) +2. FOR EACH PDF document: + a. AI call to extract expense data in CSV format + b. If 0 records: move to "error" folder + c. Validate/calculate VAT, complete valuta/transactionDateTime + d. Save all records to TrusteePosition + e. Move document to "processed" subfolder with timestamp prefix +""" + +import logging +import time +import json +import csv +import io +import asyncio +from datetime import datetime, UTC +from typing import Dict, Any, List, Optional +from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum + +logger = logging.getLogger(__name__) + +# Configuration +MAX_FILES_PER_EXECUTION = 50 +ALLOWED_TAGS = ["customer", "meeting", "license", "subscription", "fuel", "food", "material"] +RATE_LIMIT_WAIT_SECONDS = 60 + + +async def getExpensesFromPdf(self, parameters: Dict[str, Any]) -> ActionResult: + """ + Extract expenses from PDF documents in SharePoint and save to TrusteePosition. + + Parameters: + - connectionReference (str): Microsoft connection label + - sharepointFolder (str): SharePoint folder path (e.g., /sites/MySite/Documents/Expenses) + - featureInstanceId (str): Feature instance ID for TrusteePosition + - prompt (str): AI prompt for content extraction + + Returns: + ActionResult with success status and processing summary + """ + operationId = None + processedDocuments = [] + skippedDocuments = [] + errorDocuments = [] + totalPositions = 0 + + try: + # Initialize progress tracking + workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" + operationId = f"sharepoint_expenses_{workflowId}_{int(time.time())}" + + parentOperationId = parameters.get('parentOperationId') + self.services.chat.progressLogStart( + operationId, + "Extract Expenses from PDF", + "SharePoint PDF Processing", + "Initializing expense extraction", + parentOperationId=parentOperationId + ) + + # Extract and validate parameters + connectionReference = parameters.get("connectionReference") + sharepointFolder = parameters.get("sharepointFolder") + featureInstanceId = parameters.get("featureInstanceId") + prompt = parameters.get("prompt") + + if not connectionReference: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="connectionReference is required") + if not sharepointFolder: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="sharepointFolder is required") + if not featureInstanceId: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="featureInstanceId is required") + if not prompt: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="prompt is required") + + # Get Microsoft connection + self.services.chat.progressLogUpdate(operationId, 0.05, "Getting Microsoft connection") + connection = self.connection.getMicrosoftConnection(connectionReference) + if not connection: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="No valid Microsoft connection found") + + # Find site and folder info + self.services.chat.progressLogUpdate(operationId, 0.1, "Resolving SharePoint site") + siteInfo, folderPath = await _resolveSiteAndFolder(self, sharepointFolder) + if not siteInfo: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error=f"Could not resolve SharePoint site from path: {sharepointFolder}") + + siteId = siteInfo.get("id") + + # List PDF files in folder + self.services.chat.progressLogUpdate(operationId, 0.15, "Finding PDF files in folder") + pdfFiles = await _listPdfFilesInFolder(self, siteId, folderPath) + + if not pdfFiles: + self.services.chat.progressLogFinish(operationId, True) + return ActionResult.isSuccess( + documents=[ActionDocument( + documentName="expense_extraction_result.json", + documentData=json.dumps({ + "status": "no_documents", + "message": "No PDF files found in the specified folder", + "folder": sharepointFolder + }, indent=2), + mimeType="application/json", + validationMetadata={"actionType": "sharepoint.getExpensesFromPdf"} + )] + ) + + # Limit files + originalFileCount = len(pdfFiles) + if originalFileCount > MAX_FILES_PER_EXECUTION: + logger.warning(f"Found {originalFileCount} PDFs, limiting to {MAX_FILES_PER_EXECUTION}") + pdfFiles = pdfFiles[:MAX_FILES_PER_EXECUTION] + + totalFiles = len(pdfFiles) + progressPerFile = 0.7 / totalFiles + + # Get Trustee interface + from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface + trusteeInterface = getTrusteeInterface( + self.services.user, + mandateId=self.services.mandateId, + featureInstanceId=featureInstanceId + ) + + # Process each PDF + for idx, pdfFile in enumerate(pdfFiles): + currentProgress = 0.2 + (idx * progressPerFile) + fileName = pdfFile.get("name", f"file_{idx}") + fileId = pdfFile.get("id") + + self.services.chat.progressLogUpdate( + operationId, + currentProgress, + f"Processing {idx + 1}/{totalFiles}: {fileName}" + ) + + try: + # Download PDF content + fileContent = await self.services.sharepoint.downloadFile(siteId, fileId) + if not fileContent: + await _moveToErrorFolder(self, siteId, folderPath, fileName) + errorDocuments.append({ + "file": fileName, + "error": "Failed to download", + "movedTo": "error/" + }) + continue + + # AI call to extract expense data + aiResult = await _extractExpensesWithAi(self.services, fileContent, fileName, prompt, featureInstanceId) + + if not aiResult.get("success"): + await _moveToErrorFolder(self, siteId, folderPath, fileName) + errorDocuments.append({ + "file": fileName, + "error": aiResult.get("error", "AI extraction failed"), + "movedTo": "error/" + }) + continue + + records = aiResult.get("records", []) + + # Check for empty records + if not records: + logger.warning(f"Document {fileName}: No records extracted, moving to error folder") + await _moveToErrorFolder(self, siteId, folderPath, fileName) + skippedDocuments.append({ + "file": fileName, + "reason": "No expense records extracted", + "movedTo": "error/" + }) + continue + + # Validate and enrich records + validatedRecords = _validateAndEnrichRecords(records, fileName) + + # Save to TrusteePosition + savedCount = _saveToTrusteePosition(trusteeInterface, validatedRecords, featureInstanceId, self.services.mandateId) + totalPositions += savedCount + + # Move document to "processed" subfolder + timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + newFileName = f"{timestamp}_{fileName}" + + moveSuccess = await _moveToProcessedFolder(self, siteId, folderPath, fileName, newFileName) + + processedDocuments.append({ + "file": fileName, + "newLocation": f"processed/{newFileName}" if moveSuccess else "move_failed", + "recordsExtracted": len(validatedRecords), + "recordsSaved": savedCount + }) + + except Exception as e: + errorMsg = str(e) + logger.error(f"Error processing {fileName}: {errorMsg}") + + # Handle rate limit + if "429" in errorMsg or "throttl" in errorMsg.lower(): + logger.warning(f"Rate limit hit, waiting {RATE_LIMIT_WAIT_SECONDS} seconds") + await asyncio.sleep(RATE_LIMIT_WAIT_SECONDS) + + await _moveToErrorFolder(self, siteId, folderPath, fileName) + errorDocuments.append({ + "file": fileName, + "error": errorMsg, + "movedTo": "error/" + }) + + # Create result summary + self.services.chat.progressLogUpdate(operationId, 0.95, "Creating result summary") + + remainingFiles = max(0, originalFileCount - MAX_FILES_PER_EXECUTION) + + resultSummary = { + "status": "completed", + "folder": sharepointFolder, + "featureInstanceId": featureInstanceId, + "summary": { + "totalFilesFound": originalFileCount, + "filesProcessedThisRun": totalFiles, + "remainingFiles": remainingFiles, + "successfulDocuments": len(processedDocuments), + "skippedDocuments": len(skippedDocuments), + "errorDocuments": len(errorDocuments), + "totalPositionsSaved": totalPositions + }, + "processedDocuments": processedDocuments, + "skippedDocuments": skippedDocuments, + "errorDocuments": errorDocuments + } + + if remainingFiles > 0: + resultSummary["note"] = f"{remainingFiles} files remaining for next execution" + + self.services.chat.progressLogFinish(operationId, True) + + return ActionResult.isSuccess( + documents=[ActionDocument( + documentName="expense_extraction_result.json", + documentData=json.dumps(resultSummary, indent=2), + mimeType="application/json", + validationMetadata={ + "actionType": "sharepoint.getExpensesFromPdf", + "sharepointFolder": sharepointFolder, + "featureInstanceId": featureInstanceId, + "totalPositions": totalPositions + } + )] + ) + + except Exception as e: + logger.error(f"Error in getExpensesFromPdf: {str(e)}") + if operationId: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error=str(e)) + + +async def _resolveSiteAndFolder(self, sharepointFolder: str) -> tuple: + """Resolve SharePoint site and folder path from the given path.""" + try: + # Parse path format: /sites/SiteName/FolderPath + if sharepointFolder.startswith('/sites/'): + parts = sharepointFolder[7:].split('/', 1) # Remove '/sites/' prefix + if len(parts) >= 1: + siteName = parts[0] + folderPath = parts[1] if len(parts) > 1 else "" + + # Try to find site by name + sites, _ = await self.siteDiscovery.resolveSitesFromPathQuery(sharepointFolder) + if sites: + return sites[0], folderPath + + # Fallback: try to resolve via siteDiscovery + sites, _ = await self.siteDiscovery.resolveSitesFromPathQuery(sharepointFolder) + if sites: + return sites[0], "" + + return None, None + + except Exception as e: + logger.error(f"Error resolving site and folder: {str(e)}") + return None, None + + +async def _listPdfFilesInFolder(self, siteId: str, folderPath: str) -> List[Dict[str, Any]]: + """List PDF files in the given folder.""" + try: + import urllib.parse + + # Build endpoint + if not folderPath or folderPath == "/": + endpoint = f"sites/{siteId}/drive/root/children" + else: + cleanPath = folderPath.strip('/') + encodedPath = urllib.parse.quote(cleanPath, safe='/') + endpoint = f"sites/{siteId}/drive/root:/{encodedPath}:/children" + + result = await self.apiClient.makeGraphApiCall(endpoint) + + if "error" in result: + logger.error(f"Error listing folder: {result['error']}") + return [] + + items = result.get("value", []) + + # Filter for PDF files only + pdfFiles = [] + for item in items: + name = item.get("name", "") + if name.lower().endswith('.pdf') and "file" in item: + pdfFiles.append({ + "id": item.get("id"), + "name": name, + "size": item.get("size", 0), + "webUrl": item.get("webUrl"), + "lastModifiedDateTime": item.get("lastModifiedDateTime") + }) + + logger.info(f"Found {len(pdfFiles)} PDF files in folder") + return pdfFiles + + except Exception as e: + logger.error(f"Error listing PDF files: {str(e)}") + return [] + + +async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, prompt: str, featureInstanceId: str) -> Dict[str, Any]: + """ + Call AI service to extract expense data from PDF content. + Uses the full AI service pipeline which handles: + - Document extraction (text + images) + - Intent analysis + - Chunking for large documents + - Vision processing for images + """ + try: + import uuid + + # Ensure AI is initialized + await services.ai.ensureAiObjectsInitialized() + + # Step 1: Store file temporarily in database so AI service can access it + from modules.interfaces.interfaceDbManagement import getInterface as getDbInterface + from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelDocref import DocumentReferenceList + + dbInterface = getDbInterface() + + # Create file record + fileItem = dbInterface.createFile( + name=fileName, + mimeType="application/pdf", + content=fileContent + ) + + # Store file data + dbInterface.createFileData(fileItem.id, fileContent) + + logger.info(f"Stored PDF {fileName} ({len(fileContent)} bytes) with fileId: {fileItem.id}") + + # Step 2: Create ChatDocument referencing the file + # Use workflow context if available + workflowId = services.workflow.id if services.workflow else str(uuid.uuid4()) + messageId = f"expense_import_{workflowId}_{str(uuid.uuid4())[:8]}" + + chatDocument = ChatDocument( + id=str(uuid.uuid4()), + mandateId=services.mandateId or "", + featureInstanceId=featureInstanceId or "", + messageId=messageId, + fileId=fileItem.id, + fileName=fileName, + fileSize=len(fileContent), + mimeType="application/pdf" + ) + + # Step 3: Create DocumentReferenceList for AI service + from modules.datamodels.datamodelDocref import DocumentItemReference + documentList = DocumentReferenceList( + references=[ + DocumentItemReference( + documentId=chatDocument.id, + fileName=fileName + ) + ] + ) + + # Step 4: Store the ChatDocument so AI service can retrieve it + # The AI service uses getChatDocumentsFromDocumentList which queries the database + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + chatInterface = getChatInterface(services.user, mandateId=services.mandateId, featureInstanceId=featureInstanceId) + chatInterface.createDocument(chatDocument.model_dump()) + + logger.info(f"Created ChatDocument {chatDocument.id} for AI processing") + + # Step 5: Call AI with documentList - let AI service handle everything + # (extraction, intent analysis, chunking, image processing) + options = AiCallOptions( + resultFormat="csv", + operationType=OperationTypeEnum.DATA_EXTRACT + ) + + aiResponse = await services.ai.callAiContent( + prompt=prompt, + options=options, + documentList=documentList, + contentParts=None, # Let AI service extract from documents + outputFormat="csv" + ) + + if not aiResponse or not aiResponse.content: + return {"success": False, "error": "AI returned empty response"} + + # Parse CSV response + csvContent = aiResponse.content + records = _parseCsvToRecords(csvContent) + + return {"success": True, "records": records} + + except Exception as e: + logger.error(f"AI extraction error for {fileName}: {str(e)}") + return {"success": False, "error": str(e)} + + +def _parseCsvToRecords(csvContent: str) -> List[Dict[str, Any]]: + """Parse CSV content to list of expense records.""" + records = [] + try: + # Clean up CSV content - remove markdown code blocks if present + content = csvContent.strip() + if content.startswith("```"): + lines = content.split('\n') + # Remove first and last line if they're code block markers + if lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + content = '\n'.join(lines) + + reader = csv.DictReader(io.StringIO(content)) + for row in reader: + # Clean up keys (remove whitespace) + cleanedRow = {k.strip(): v.strip() if isinstance(v, str) else v for k, v in row.items()} + records.append(cleanedRow) + + except Exception as e: + logger.error(f"Error parsing CSV: {str(e)}") + + return records + + +def _validateAndEnrichRecords(records: List[Dict[str, Any]], sourceFileName: str) -> List[Dict[str, Any]]: + """ + Validate and enrich expense records: + 1. Calculate/correct VAT amount + 2. Complete valuta/transactionDateTime if one is missing + 3. Validate tags + """ + enrichedRecords = [] + + for record in records: + enriched = record.copy() + + # VAT calculation/validation + vatPercentage = _parseFloat(record.get("vatPercentage", 0)) + vatAmount = _parseFloat(record.get("vatAmount", 0)) + bookingAmount = _parseFloat(record.get("bookingAmount", 0)) + + if vatPercentage > 0 and bookingAmount > 0: + # Calculate expected VAT amount (VAT is included in bookingAmount) + expectedVat = bookingAmount * vatPercentage / (100 + vatPercentage) + + # If vatAmount is missing or significantly different, recalculate + if vatAmount == 0 or abs(vatAmount - expectedVat) > 0.01: + enriched["vatAmount"] = round(expectedVat, 2) + logger.debug(f"VAT amount corrected: {vatAmount} -> {enriched['vatAmount']}") + + # Valuta / transactionDateTime completion + valuta = record.get("valuta") + transactionDateTime = record.get("transactionDateTime") + + if valuta and not transactionDateTime: + try: + dt = datetime.strptime(str(valuta).strip(), "%Y-%m-%d") + enriched["transactionDateTime"] = dt.replace(hour=12).timestamp() + except: + pass + elif transactionDateTime and not valuta: + try: + ts = float(transactionDateTime) + dt = datetime.fromtimestamp(ts, UTC) + enriched["valuta"] = dt.strftime("%Y-%m-%d") + except: + pass + + # Validate tags + tags = record.get("tags", "") + if tags: + tagList = [t.strip().lower() for t in str(tags).split(",")] + validTags = [t for t in tagList if t in ALLOWED_TAGS] + enriched["tags"] = ",".join(validTags) + + # Store source file info in description + existingDesc = record.get("desc", "") + if sourceFileName and sourceFileName not in str(existingDesc): + enriched["desc"] = f"[Source: {sourceFileName}]\n{existingDesc}" + + enrichedRecords.append(enriched) + + return enrichedRecords + + +def _parseFloat(value) -> float: + """Safely parse float value.""" + try: + if value is None or value == "": + return 0.0 + return float(value) + except (ValueError, TypeError): + return 0.0 + + +def _saveToTrusteePosition(trusteeInterface, records: List[Dict[str, Any]], featureInstanceId: str, mandateId: str) -> int: + """Save validated records to TrusteePosition table.""" + savedCount = 0 + + for record in records: + try: + position = { + "valuta": record.get("valuta"), + "transactionDateTime": record.get("transactionDateTime"), + "company": record.get("company", ""), + "desc": record.get("desc", ""), + "tags": record.get("tags", ""), + "bookingCurrency": record.get("bookingCurrency", "CHF"), + "bookingAmount": _parseFloat(record.get("bookingAmount", 0)), + "originalCurrency": record.get("originalCurrency") or record.get("bookingCurrency", "CHF"), + "originalAmount": _parseFloat(record.get("originalAmount", 0)) or _parseFloat(record.get("bookingAmount", 0)), + "vatPercentage": _parseFloat(record.get("vatPercentage", 0)), + "vatAmount": _parseFloat(record.get("vatAmount", 0)), + "featureInstanceId": featureInstanceId, + "mandateId": mandateId + } + + result = trusteeInterface.createPosition(position) + if result: + savedCount += 1 + logger.debug(f"Saved position: {position.get('company')} - {position.get('bookingAmount')}") + + except Exception as e: + logger.error(f"Failed to save position: {str(e)}") + + return savedCount + + +async def _ensureFolderExists(self, siteId: str, folderPath: str) -> bool: + """Create folder if it doesn't exist.""" + try: + import urllib.parse + + # Check if folder exists + cleanPath = folderPath.strip('/') + encodedPath = urllib.parse.quote(cleanPath, safe='/') + checkEndpoint = f"sites/{siteId}/drive/root:/{encodedPath}" + + result = await self.apiClient.makeGraphApiCall(checkEndpoint) + + if "error" not in result: + return True # Folder exists + + # Create folder - need to create parent first if nested + pathParts = cleanPath.split('/') + currentPath = "" + + for part in pathParts: + parentPath = currentPath if currentPath else "root" + currentPath = f"{currentPath}/{part}" if currentPath else part + + # Check if this level exists + checkPath = urllib.parse.quote(currentPath, safe='/') + checkResult = await self.apiClient.makeGraphApiCall(f"sites/{siteId}/drive/root:/{checkPath}") + + if "error" in checkResult: + # Create this folder + if parentPath == "root": + createEndpoint = f"sites/{siteId}/drive/root/children" + else: + encodedParent = urllib.parse.quote(parentPath, safe='/') + createEndpoint = f"sites/{siteId}/drive/root:/{encodedParent}:/children" + + createData = json.dumps({ + "name": part, + "folder": {}, + "@microsoft.graph.conflictBehavior": "fail" + }).encode('utf-8') + + createResult = await self.apiClient.makeGraphApiCall(createEndpoint, method="POST", data=createData) + + if "error" in createResult: + logger.warning(f"Failed to create folder {part}: {createResult['error']}") + return False + + logger.info(f"Created folder: {currentPath}") + + return True + + except Exception as e: + logger.error(f"Failed to ensure folder exists: {str(e)}") + return False + + +async def _moveToProcessedFolder(self, siteId: str, sourceFolderPath: str, sourceFileName: str, destFileName: str) -> bool: + """Move processed PDF to 'processed' subfolder.""" + try: + # Build processed folder path + cleanSource = sourceFolderPath.strip('/') + processedFolder = f"{cleanSource}/processed" if cleanSource else "processed" + + # Ensure processed folder exists + await _ensureFolderExists(self, siteId, processedFolder) + + # Copy file to new location + await self.services.sharepoint.copyFileAsync( + siteId=siteId, + sourceFolder=cleanSource if cleanSource else "/", + sourceFile=sourceFileName, + destFolder=processedFolder, + destFile=destFileName + ) + + # Delete original file + await _deleteFile(self, siteId, sourceFolderPath, sourceFileName) + + logger.info(f"Moved {sourceFileName} to processed/{destFileName}") + return True + + except Exception as e: + logger.error(f"Failed to move file to processed: {str(e)}") + return False + + +async def _moveToErrorFolder(self, siteId: str, sourceFolderPath: str, sourceFileName: str) -> bool: + """Move failed PDF to 'error' subfolder (filename unchanged).""" + try: + # Build error folder path + cleanSource = sourceFolderPath.strip('/') + errorFolder = f"{cleanSource}/error" if cleanSource else "error" + + # Ensure error folder exists + await _ensureFolderExists(self, siteId, errorFolder) + + # Copy file to error folder (keep original name) + await self.services.sharepoint.copyFileAsync( + siteId=siteId, + sourceFolder=cleanSource if cleanSource else "/", + sourceFile=sourceFileName, + destFolder=errorFolder, + destFile=sourceFileName # Same filename + ) + + # Delete original file + await _deleteFile(self, siteId, sourceFolderPath, sourceFileName) + + logger.info(f"Moved {sourceFileName} to error/") + return True + + except Exception as e: + logger.error(f"Failed to move file to error folder: {str(e)}") + return False + + +async def _deleteFile(self, siteId: str, folderPath: str, fileName: str) -> bool: + """Delete file from SharePoint.""" + try: + import urllib.parse + + cleanPath = folderPath.strip('/') + filePath = f"{cleanPath}/{fileName}" if cleanPath else fileName + encodedPath = urllib.parse.quote(filePath, safe='/') + + endpoint = f"sites/{siteId}/drive/root:/{encodedPath}" + + # Get file ID first + fileInfo = await self.apiClient.makeGraphApiCall(endpoint) + if "error" in fileInfo: + logger.warning(f"File not found for deletion: {filePath}") + return False + + fileId = fileInfo.get("id") + if not fileId: + return False + + # Delete by ID + deleteEndpoint = f"sites/{siteId}/drive/items/{fileId}" + + # Make DELETE request + if self.services.sharepoint.accessToken is None: + logger.error("Access token not set for delete") + return False + + import aiohttp + headers = {"Authorization": f"Bearer {self.services.sharepoint.accessToken}"} + url = f"https://graph.microsoft.com/v1.0/{deleteEndpoint}" + + async with aiohttp.ClientSession() as session: + async with session.delete(url, headers=headers) as response: + if response.status in [200, 204]: + logger.debug(f"Deleted file: {filePath}") + return True + else: + errorText = await response.text() + logger.warning(f"Delete failed: {response.status} - {errorText}") + return False + + except Exception as e: + logger.error(f"Failed to delete file: {str(e)}") + return False diff --git a/modules/workflows/methods/methodSharepoint/methodSharepoint.py b/modules/workflows/methods/methodSharepoint/methodSharepoint.py index e8d41905..5b765fe5 100644 --- a/modules/workflows/methods/methodSharepoint/methodSharepoint.py +++ b/modules/workflows/methods/methodSharepoint/methodSharepoint.py @@ -28,6 +28,7 @@ from .actions.findSiteByUrl import findSiteByUrl from .actions.downloadFileByPath import downloadFileByPath from .actions.copyFile import copyFile from .actions.uploadFile import uploadFile +from .actions.getExpensesFromPdf import getExpensesFromPdf logger = logging.getLogger(__name__) @@ -377,6 +378,42 @@ class MethodSharepoint(MethodBase): ) }, execute=uploadFile.__get__(self, self.__class__) + ), + "getExpensesFromPdf": WorkflowActionDefinition( + actionId="sharepoint.getExpensesFromPdf", + description="Extract expenses from PDF documents in SharePoint folder and save to TrusteePosition", + dynamicMode=False, # Not for dynamic workflow + parameters={ + "connectionReference": WorkflowActionParameter( + name="connectionReference", + type="str", + frontendType=FrontendType.USER_CONNECTION, + required=True, + description="Microsoft connection label for SharePoint access" + ), + "sharepointFolder": WorkflowActionParameter( + name="sharepointFolder", + type="str", + frontendType=FrontendType.TEXT, + required=True, + description="SharePoint folder path containing PDF expense documents (e.g., /sites/MySite/Documents/Expenses)" + ), + "featureInstanceId": WorkflowActionParameter( + name="featureInstanceId", + type="str", + frontendType=FrontendType.TEXT, + required=True, + description="Feature Instance ID for the Trustee feature where positions will be stored" + ), + "prompt": WorkflowActionParameter( + name="prompt", + type="str", + frontendType=FrontendType.TEXTAREA, + required=True, + description="AI prompt for extracting expense data from PDF content" + ) + }, + execute=getExpensesFromPdf.__get__(self, self.__class__) ) } @@ -393,4 +430,5 @@ class MethodSharepoint(MethodBase): self.downloadFileByPath = downloadFileByPath.__get__(self, self.__class__) self.copyFile = copyFile.__get__(self, self.__class__) self.uploadFile = uploadFile.__get__(self, self.__class__) + self.getExpensesFromPdf = getExpensesFromPdf.__get__(self, self.__class__) diff --git a/tests/unit/rbac/test_rbac_bootstrap.py b/tests/unit/rbac/test_rbac_bootstrap.py index 476efab1..05951264 100644 --- a/tests/unit/rbac/test_rbac_bootstrap.py +++ b/tests/unit/rbac/test_rbac_bootstrap.py @@ -119,33 +119,27 @@ class TestRbacBootstrap: # Should create multiple rules for different tables assert db.recordCreate.call_count > 0 - # Check that Mandate table rules are created + # Check that Mandate table rules are created with full objectKey mandateCalls = [call for call in db.recordCreate.call_args_list - if call[0][1].item == "Mandate"] + if call[0][1].item == "data.system.Mandate"] assert len(mandateCalls) > 0 - # Check sysadmin rule for Mandate - sysadminMandateCall = [call for call in mandateCalls - if call[0][1].roleLabel == "sysadmin"][0] - sysadminRule = sysadminMandateCall[0][1] - assert sysadminRule.view == True - assert sysadminRule.read == AccessLevel.ALL - - # Check that other roles have view=False for Mandate - otherMandateCalls = [call for call in mandateCalls - if call[0][1].roleLabel != "sysadmin"] - for call in otherMandateCalls: + # Check that all roles have view=False and no access for Mandate + # (SysAdmin bypasses RBAC via isSysAdmin flag, not via roles) + for call in mandateCalls: rule = call[0][1] assert rule.view == False + assert rule.read == AccessLevel.NONE def testInitRbacRulesSkipsIfExists(self): """Test that initRbacRules skips default rule creation if rules already exist, but adds missing table-specific rules.""" db = Mock() # Mock existing rules - include rules for ChatWorkflow and Prompt to prevent adding missing rules # Need rules for all required roles to fully prevent creation + # Using full objectKey format: data.system.{TableName} existingRules = [] - for table in ["ChatWorkflow", "Prompt"]: - for role in ["sysadmin", "admin", "user", "viewer"]: + for table in ["data.system.ChatWorkflow", "data.system.Prompt"]: + for role in ["admin", "user", "viewer"]: existingRules.append({ "id": f"rule_{table}_{role}", "item": table, diff --git a/tests/unit/rbac/test_rbac_permissions.py b/tests/unit/rbac/test_rbac_permissions.py index d063ff11..a3387f92 100644 --- a/tests/unit/rbac/test_rbac_permissions.py +++ b/tests/unit/rbac/test_rbac_permissions.py @@ -94,7 +94,7 @@ class TestRbacPermissionResolution: AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB", # Specific rule + item="data.system.UserInDB", # Specific rule with full objectKey view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -107,10 +107,11 @@ class TestRbacPermissionResolution: rbac._getRulesForRole = mockGetRulesForRole # Get permissions for UserInDB table - should use specific rule + # Using full objectKey format: data.system.UserInDB permissions = rbac.getUserPermissions( user, AccessRuleContext.DATA, - "UserInDB" + "data.system.UserInDB" ) # Most specific rule should win @@ -252,29 +253,29 @@ class TestRbacPermissionResolution: AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB", # Table-level + item="data.system.UserInDB", # Table-level with full objectKey view=True, read=AccessLevel.MY ), AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB.email", # Field-level - most specific + item="data.system.UserInDB.email", # Field-level - most specific view=True, read=AccessLevel.NONE ) ] # Test exact match - rule = rbac.findMostSpecificRule(rules, "UserInDB.email") + rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB.email") assert rule is not None - assert rule.item == "UserInDB.email" + assert rule.item == "data.system.UserInDB.email" assert rule.read == AccessLevel.NONE # Test table-level match - rule = rbac.findMostSpecificRule(rules, "UserInDB") + rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB") assert rule is not None - assert rule.item == "UserInDB" + assert rule.item == "data.system.UserInDB" assert rule.read == AccessLevel.MY # Test generic fallback @@ -293,7 +294,7 @@ class TestRbacPermissionResolution: rule1 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -306,7 +307,7 @@ class TestRbacPermissionResolution: rule2 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.GROUP, # Not allowed @@ -319,7 +320,7 @@ class TestRbacPermissionResolution: rule3 = AccessRule( roleLabel="admin", context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -332,7 +333,7 @@ class TestRbacPermissionResolution: rule4 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="UserInDB", + item="data.system.UserInDB", view=True, read=AccessLevel.NONE, create=AccessLevel.MY, # Not allowed without read From a0304c6d78baeaff9b413c870999d8ca6bc92f6b Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 01:29:17 +0100 Subject: [PATCH 26/32] mandate invitation and notification system --- app.py | 3 + modules/connectors/connectorDbPostgre.py | 48 +- modules/datamodels/datamodelInvitation.py | 17 +- modules/datamodels/datamodelNotification.py | 209 +++++++ modules/routes/routeAdminFeatures.py | 53 +- modules/routes/routeInvitations.py | 152 +++++- modules/routes/routeNotifications.py | 575 ++++++++++++++++++++ modules/routes/routeSecurityLocal.py | 47 ++ modules/system/registry.py | 16 +- 9 files changed, 1072 insertions(+), 48 deletions(-) create mode 100644 modules/datamodels/datamodelNotification.py create mode 100644 modules/routes/routeNotifications.py diff --git a/app.py b/app.py index dec478fc..a6f07f33 100644 --- a/app.py +++ b/app.py @@ -492,6 +492,9 @@ app.include_router(featuresAdminRouter) from modules.routes.routeInvitations import router as invitationsRouter app.include_router(invitationsRouter) +from modules.routes.routeNotifications import router as notificationsRouter +app.include_router(notificationsRouter) + from modules.routes.routeAdminRbacExport import router as rbacAdminExportRouter app.include_router(rbacAdminExportRouter) diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 2dfec2b4..6c89a85f 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -40,6 +40,34 @@ class SystemTable(BaseModel): ) +def _isJsonbType(fieldType) -> bool: + """Check if a type should be stored as JSONB in PostgreSQL.""" + # Direct dict or list + if fieldType == dict or fieldType == list: + return True + + # Generic List[X] or Dict[X, Y] + origin = get_origin(fieldType) + if origin in (dict, list): + return True + + # Direct Pydantic BaseModel subclass + if isinstance(fieldType, type) and issubclass(fieldType, BaseModel): + return True + + # Optional[X] - check the inner type + if origin is Union: + args = get_args(fieldType) + for arg in args: + if arg is type(None): + continue + # Recursively check the inner type + if _isJsonbType(arg): + return True + + return False + + def _get_model_fields(model_class) -> Dict[str, str]: """Get all fields from Pydantic model and map to SQL types.""" # Pydantic v2 @@ -52,20 +80,7 @@ def _get_model_fields(model_class) -> Dict[str, str]: # Check for JSONB fields (Dict, List, or complex types) # Purely type-based detection - no hardcoded field names - if ( - field_type == dict - or field_type == list - or ( - hasattr(field_type, "__origin__") - and field_type.__origin__ in (dict, list) - ) - # Check if field type is directly a Pydantic BaseModel subclass (for nested models like TextMultilingual) - or (isinstance(field_type, type) and issubclass(field_type, BaseModel)) - # Check if field type is Optional[BaseModel] (Union with None) - or (hasattr(field_type, "__origin__") and get_origin(field_type) is Union - and any(isinstance(arg, type) and issubclass(arg, BaseModel) - for arg in get_args(field_type) if arg is not type(None))) - ): + if _isJsonbType(field_type): fields[field_name] = "JSONB" # Simple type mapping elif field_type in (str, type(None)) or ( @@ -970,7 +985,10 @@ class DatabaseConnector: record["id"] = str(uuid.uuid4()) # Save record - self._saveRecord(model_class, record["id"], record) + success = self._saveRecord(model_class, record["id"], record) + if not success: + table = model_class.__name__ + raise ValueError(f"Failed to save record {record['id']} to table {table}") # Check if this is the first record in the table and register as initial ID table = model_class.__name__ diff --git a/modules/datamodels/datamodelInvitation.py b/modules/datamodels/datamodelInvitation.py index a35dfb09..ef6d6a80 100644 --- a/modules/datamodels/datamodelInvitation.py +++ b/modules/datamodels/datamodelInvitation.py @@ -46,9 +46,13 @@ class Invitation(BaseModel): ) # Einladungs-Details + targetUsername: str = Field( + description="Username of the invited user (must match on acceptance)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} + ) email: Optional[str] = Field( default=None, - description="Target email address (optional, for tracking)", + description="Email address to send invitation link (optional)", json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": False} ) createdBy: str = Field( @@ -82,6 +86,13 @@ class Invitation(BaseModel): json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} ) + # Email-Status + emailSent: bool = Field( + default=False, + description="Whether the invitation email was successfully sent", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": False} + ) + # Einschränkungen maxUses: int = Field( default=1, @@ -107,13 +118,15 @@ registerModelLabels( "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, "featureInstanceId": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance"}, "roleIds": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, - "email": {"en": "Email", "de": "E-Mail", "fr": "Email"}, + "targetUsername": {"en": "Target Username", "de": "Ziel-Benutzername", "fr": "Nom d'utilisateur cible"}, + "email": {"en": "Email (optional)", "de": "E-Mail (optional)", "fr": "Email (optionnel)"}, "createdBy": {"en": "Created By", "de": "Erstellt von", "fr": "Créé par"}, "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, "expiresAt": {"en": "Expires At", "de": "Gültig bis", "fr": "Expire le"}, "usedBy": {"en": "Used By", "de": "Verwendet von", "fr": "Utilisé par"}, "usedAt": {"en": "Used At", "de": "Verwendet am", "fr": "Utilisé le"}, "revokedAt": {"en": "Revoked At", "de": "Widerrufen am", "fr": "Révoqué le"}, + "emailSent": {"en": "Email Sent", "de": "E-Mail gesendet", "fr": "Email envoyé"}, "maxUses": {"en": "Max Uses", "de": "Max. Verwendungen", "fr": "Utilisations max"}, "currentUses": {"en": "Current Uses", "de": "Aktuelle Verwendungen", "fr": "Utilisations actuelles"}, }, diff --git a/modules/datamodels/datamodelNotification.py b/modules/datamodels/datamodelNotification.py new file mode 100644 index 00000000..b1475767 --- /dev/null +++ b/modules/datamodels/datamodelNotification.py @@ -0,0 +1,209 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Notification model for in-app notifications. +Supports actionable notifications (e.g., invitation accept/decline). +""" + +import uuid +from typing import Optional, List +from enum import Enum +from pydantic import BaseModel, Field, ConfigDict +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp + + +class NotificationType(str, Enum): + """Types of notifications""" + INVITATION = "invitation" # Einladung zu Mandat/Feature + SYSTEM = "system" # System-Nachrichten + WORKFLOW = "workflow" # Workflow-Status Updates + MENTION = "mention" # Erwähnung in Chat/Kommentar + + +class NotificationStatus(str, Enum): + """Status of a notification""" + UNREAD = "unread" # Noch nicht gelesen + READ = "read" # Gelesen + ACTIONED = "actioned" # Aktion wurde durchgeführt + DISMISSED = "dismissed" # Verworfen/Geschlossen + + +class NotificationAction(BaseModel): + """Possible action for a notification""" + actionId: str = Field( + description="Unique identifier for the action (e.g., 'accept', 'decline')" + ) + label: str = Field( + description="Display label for the action button" + ) + style: str = Field( + default="default", + description="Button style: 'primary', 'danger', 'default'" + ) + + +class UserNotification(BaseModel): + """ + In-app notification for a user. + Supports actionable notifications with accept/decline buttons. + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique ID of the notification", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + userId: str = Field( + description="Target user ID for this notification", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + # Notification type and status + type: NotificationType = Field( + default=NotificationType.SYSTEM, + description="Type of notification", + json_schema_extra={ + "frontend_type": "select", + "frontend_readonly": True, + "frontend_required": True, + "frontend_options": [ + {"value": "invitation", "label": {"en": "Invitation", "de": "Einladung"}}, + {"value": "system", "label": {"en": "System", "de": "System"}}, + {"value": "workflow", "label": {"en": "Workflow", "de": "Workflow"}}, + {"value": "mention", "label": {"en": "Mention", "de": "Erwähnung"}} + ] + } + ) + status: NotificationStatus = Field( + default=NotificationStatus.UNREAD, + description="Current status of the notification", + json_schema_extra={ + "frontend_type": "select", + "frontend_readonly": True, + "frontend_required": False, + "frontend_options": [ + {"value": "unread", "label": {"en": "Unread", "de": "Ungelesen"}}, + {"value": "read", "label": {"en": "Read", "de": "Gelesen"}}, + {"value": "actioned", "label": {"en": "Actioned", "de": "Bearbeitet"}}, + {"value": "dismissed", "label": {"en": "Dismissed", "de": "Verworfen"}} + ] + } + ) + + # Content + title: str = Field( + description="Notification title", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + message: str = Field( + description="Notification message/body", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": True} + ) + icon: Optional[str] = Field( + default=None, + description="Optional icon identifier (e.g., 'mail', 'warning', 'info')", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Reference to triggering object (for actionable notifications) + referenceType: Optional[str] = Field( + default=None, + description="Type of referenced object (e.g., 'Invitation', 'Workflow')", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + referenceId: Optional[str] = Field( + default=None, + description="ID of referenced object", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Actions (for actionable notifications like invitations) + actions: Optional[List[NotificationAction]] = Field( + default=None, + description="List of possible actions for this notification", + json_schema_extra={"frontend_type": "json", "frontend_readonly": True, "frontend_required": False} + ) + + # Action result (when user takes action) + actionTaken: Optional[str] = Field( + default=None, + description="Which action was taken (actionId)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + actionResult: Optional[str] = Field( + default=None, + description="Result message from the action", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} + ) + + # Timestamps + createdAt: float = Field( + default_factory=getUtcTimestamp, + description="When the notification was created (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + readAt: Optional[float] = Field( + default=None, + description="When the notification was read (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + actionedAt: Optional[float] = Field( + default=None, + description="When action was taken (UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + expiresAt: Optional[float] = Field( + default=None, + description="When the notification expires (optional, UTC timestamp)", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) + + model_config = ConfigDict(use_enum_values=True) + + +registerModelLabels( + "UserNotification", + {"en": "Notification", "de": "Benachrichtigung", "fr": "Notification"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, + "type": {"en": "Type", "de": "Typ", "fr": "Type"}, + "status": {"en": "Status", "de": "Status", "fr": "Statut"}, + "title": {"en": "Title", "de": "Titel", "fr": "Titre"}, + "message": {"en": "Message", "de": "Nachricht", "fr": "Message"}, + "icon": {"en": "Icon", "de": "Symbol", "fr": "Icône"}, + "referenceType": {"en": "Reference Type", "de": "Referenz-Typ", "fr": "Type de référence"}, + "referenceId": {"en": "Reference ID", "de": "Referenz-ID", "fr": "ID de référence"}, + "actions": {"en": "Actions", "de": "Aktionen", "fr": "Actions"}, + "actionTaken": {"en": "Action Taken", "de": "Durchgeführte Aktion", "fr": "Action effectuée"}, + "actionResult": {"en": "Action Result", "de": "Aktions-Ergebnis", "fr": "Résultat de l'action"}, + "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, + "readAt": {"en": "Read At", "de": "Gelesen am", "fr": "Lu le"}, + "actionedAt": {"en": "Actioned At", "de": "Bearbeitet am", "fr": "Traité le"}, + "expiresAt": {"en": "Expires At", "de": "Gültig bis", "fr": "Expire le"}, + }, +) + + +registerModelLabels( + "NotificationType", + {"en": "Notification Type", "de": "Benachrichtigungs-Typ", "fr": "Type de notification"}, + { + "invitation": {"en": "Invitation", "de": "Einladung", "fr": "Invitation"}, + "system": {"en": "System", "de": "System", "fr": "Système"}, + "workflow": {"en": "Workflow", "de": "Workflow", "fr": "Workflow"}, + "mention": {"en": "Mention", "de": "Erwähnung", "fr": "Mention"}, + }, +) + + +registerModelLabels( + "NotificationStatus", + {"en": "Notification Status", "de": "Benachrichtigungs-Status", "fr": "Statut de notification"}, + { + "unread": {"en": "Unread", "de": "Ungelesen", "fr": "Non lu"}, + "read": {"en": "Read", "de": "Gelesen", "fr": "Lu"}, + "actioned": {"en": "Actioned", "de": "Bearbeitet", "fr": "Traité"}, + "dismissed": {"en": "Dismissed", "de": "Verworfen", "fr": "Rejeté"}, + }, +) diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 1bb6be16..82b796c1 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -21,6 +21,7 @@ from modules.datamodels.datamodelUam import User, UserInDB from modules.datamodels.datamodelFeatures import Feature, FeatureInstance from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface +from modules.security.rbacCatalog import getCatalogService logger = logging.getLogger(__name__) @@ -72,15 +73,16 @@ async def list_features( """ List all available features. - Returns global feature definitions that can be activated for mandates. + Returns global feature definitions from the RBAC Catalog. + Features are automatically registered at startup from feature containers. Any authenticated user can see available features. """ try: - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - - features = featureInterface.getAllFeatures() - return [f.model_dump() for f in features] + # Features come from the RBAC Catalog (registered at startup from feature containers) + # NOT from the database - features are code-defined, not user-created + catalogService = getCatalogService() + features = catalogService.getFeatureDefinitions() + return features except Exception as e: logger.error(f"Error listing features: {e}") @@ -153,14 +155,15 @@ async def get_my_feature_instances( "features": [] } - # Get feature info + # Get feature info from catalog (features are code-defined) featureKey = f"{mandateId}_{instance.featureCode}" if featureKey not in featuresMap: - feature = featureInterface.getFeature(instance.featureCode) + catalogService = getCatalogService() + featureDef = catalogService.getFeatureDefinition(instance.featureCode) featuresMap[featureKey] = { "code": instance.featureCode, - "label": feature.label if feature and hasattr(feature, 'label') else {"de": instance.featureCode, "en": instance.featureCode}, - "icon": feature.icon if feature and hasattr(feature, 'icon') else "folder", + "label": featureDef.get("label", {"de": instance.featureCode, "en": instance.featureCode}) if featureDef else {"de": instance.featureCode, "en": instance.featureCode}, + "icon": featureDef.get("icon", "folder") if featureDef else "folder", "instances": [], "_mandateId": mandateId # Temporary for grouping } @@ -376,8 +379,9 @@ async def create_feature( rootInterface = getRootInterface() featureInterface = getFeatureInterface(rootInterface.db) - # Check if feature already exists - existing = featureInterface.getFeature(code) + # Check if feature already exists in catalog (features are code-defined) + catalogService = getCatalogService() + existing = catalogService.getFeatureDefinition(code) if existing: raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -525,9 +529,10 @@ async def create_feature_instance( rootInterface = getRootInterface() featureInterface = getFeatureInterface(rootInterface.db) - # Verify feature exists - feature = featureInterface.getFeature(data.featureCode) - if not feature: + # Verify feature exists in catalog (features are code-defined, not DB-stored) + catalogService = getCatalogService() + featureDef = catalogService.getFeatureDefinition(data.featureCode) + if not featureDef: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Feature '{data.featureCode}' not found" @@ -818,9 +823,10 @@ async def create_template_role( rootInterface = getRootInterface() featureInterface = getFeatureInterface(rootInterface.db) - # Verify feature exists - feature = featureInterface.getFeature(featureCode) - if not feature: + # Verify feature exists in catalog (features are code-defined) + catalogService = getCatalogService() + featureDef = catalogService.getFeatureDefinition(featureCode) + if not featureDef: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Feature '{featureCode}' not found" @@ -1331,17 +1337,16 @@ async def get_feature( featureCode: Feature code (e.g., 'trustee', 'chatbot') """ try: - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - - feature = featureInterface.getFeature(featureCode) - if not feature: + # Features come from the RBAC Catalog (code-defined, not DB-stored) + catalogService = getCatalogService() + featureDef = catalogService.getFeatureDefinition(featureCode) + if not featureDef: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Feature '{featureCode}' not found" ) - return feature.model_dump() + return featureDef except HTTPException: raise diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index 47fda648..2196bd73 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -37,7 +37,8 @@ router = APIRouter( class InvitationCreate(BaseModel): """Request model for creating an invitation""" - email: Optional[str] = Field(None, description="Target email address (optional)") + targetUsername: str = Field(..., description="Username of the user to invite (must match on acceptance)") + email: Optional[str] = Field(None, description="Email address to send invitation link (optional)") roleIds: List[str] = Field(..., description="Role IDs to assign to the invited user") featureInstanceId: Optional[str] = Field(None, description="Optional feature instance access") expiresInHours: int = Field( @@ -61,6 +62,7 @@ class InvitationResponse(BaseModel): mandateId: str featureInstanceId: Optional[str] roleIds: List[str] + targetUsername: str email: Optional[str] createdBy: str createdAt: float @@ -71,6 +73,7 @@ class InvitationResponse(BaseModel): maxUses: int currentUses: int inviteUrl: str # Full URL for the invitation + emailSent: bool = False # Whether invitation email was sent class InvitationValidation(BaseModel): @@ -78,8 +81,11 @@ class InvitationValidation(BaseModel): valid: bool reason: Optional[str] mandateId: Optional[str] + mandateName: Optional[str] = None featureInstanceId: Optional[str] roleIds: List[str] + roleLabels: List[str] = [] + targetUsername: Optional[str] = None # ============================================================================= @@ -118,6 +124,11 @@ async def create_invitation( try: rootInterface = getRootInterface() + # Note: targetUsername does NOT need to exist yet! + # The invitation can be for a user who will register later. + # When they register with this username (or accept the invitation), + # they will get the assigned roles. + # Validate role IDs exist and belong to this mandate or are global for roleId in data.roleIds: from modules.datamodels.datamodelRbac import Role @@ -164,6 +175,7 @@ async def create_invitation( mandateId=str(context.mandateId), featureInstanceId=data.featureInstanceId, roleIds=data.roleIds, + targetUsername=data.targetUsername, email=data.email, createdBy=str(context.user.id), expiresAt=expiresAt, @@ -179,9 +191,98 @@ async def create_invitation( frontendUrl = APP_CONFIG.get("APP_FRONTEND_URL", "http://localhost:8080") inviteUrl = f"{frontendUrl}/invite/{invitation.token}" + # Send email if email address is provided + emailSent = False + if data.email: + try: + from modules.connectors.connectorMessagingEmail import ConnectorMessagingEmail + from modules.datamodels.datamodelUam import Mandate + + # Get mandate name for the email + mandateRecords = rootInterface.db.getRecordset( + Mandate, + recordFilter={"id": str(context.mandateId)} + ) + mandateName = mandateRecords[0].get("name", "PowerOn") if mandateRecords else "PowerOn" + + emailConnector = ConnectorMessagingEmail() + emailSubject = f"Einladung zu {mandateName}" + emailBody = f""" + + +

Sie wurden eingeladen!

+

Hallo {data.targetUsername},

+

Sie wurden eingeladen, dem Mandanten {mandateName} beizutreten.

+

Klicken Sie auf den folgenden Link, um die Einladung anzunehmen:

+

+ + Einladung annehmen + +

+

+ Oder kopieren Sie diesen Link in Ihren Browser:
+ {inviteUrl} +

+

+ Diese Einladung ist {data.expiresInHours} Stunden gültig. +

+
+

+ Diese E-Mail wurde automatisch von PowerOn gesendet. +

+ + + """ + + emailConnector.send( + recipient=data.email, + subject=emailSubject, + message=emailBody + ) + emailSent = True + logger.info(f"Invitation email sent to {data.email} for user {data.targetUsername}") + except Exception as emailError: + logger.warning(f"Failed to send invitation email to {data.email}: {emailError}") + # Don't fail the invitation creation if email fails + + # Update the invitation record with emailSent status + if emailSent: + rootInterface.db.recordModify( + Invitation, + createdRecord.get("id"), + {"emailSent": True} + ) + createdRecord["emailSent"] = True + + # If the target user already exists, create an in-app notification + try: + existingUser = rootInterface.getUserByUsername(data.targetUsername) + if existingUser: + from modules.routes.routeNotifications import createInvitationNotification + from modules.datamodels.datamodelUam import Mandate + + # Get mandate name for notification + mandateRecords = rootInterface.db.getRecordset( + Mandate, + recordFilter={"id": str(context.mandateId)} + ) + mandateName = mandateRecords[0].get("mandateLabel", "PowerOn") if mandateRecords else "PowerOn" + inviterName = context.user.fullName or context.user.username + + createInvitationNotification( + userId=str(existingUser.id), + invitationId=str(createdRecord.get("id")), + mandateName=mandateName, + inviterName=inviterName + ) + logger.info(f"Created notification for existing user {data.targetUsername}") + except Exception as notifError: + logger.warning(f"Failed to create notification for user {data.targetUsername}: {notifError}") + # Don't fail the invitation if notification fails + logger.info( - f"User {context.user.id} created invitation for mandate {context.mandateId}, " - f"expires in {data.expiresInHours}h" + f"User {context.user.id} created invitation for user {data.targetUsername} " + f"to mandate {context.mandateId}, expires in {data.expiresInHours}h" ) return InvitationResponse( @@ -190,6 +291,7 @@ async def create_invitation( mandateId=str(createdRecord.get("mandateId")), featureInstanceId=createdRecord.get("featureInstanceId"), roleIds=createdRecord.get("roleIds", []), + targetUsername=createdRecord.get("targetUsername"), email=createdRecord.get("email"), createdBy=str(createdRecord.get("createdBy")), createdAt=createdRecord.get("createdAt"), @@ -199,7 +301,8 @@ async def create_invitation( revokedAt=createdRecord.get("revokedAt"), maxUses=createdRecord.get("maxUses", 1), currentUses=createdRecord.get("currentUses", 0), - inviteUrl=inviteUrl + inviteUrl=inviteUrl, + emailSent=emailSent ) except HTTPException: @@ -441,12 +544,38 @@ async def validate_invitation( roleIds=[] ) + # Get additional info for display + mandateId = invitation.get("mandateId") + mandateName = None + roleLabels = [] + targetUsername = invitation.get("targetUsername") + + # Get mandate name + from modules.datamodels.datamodelUam import Mandate + mandateRecords = rootInterface.db.getRecordset( + Mandate, + recordFilter={"id": mandateId} + ) + if mandateRecords: + mandateName = mandateRecords[0].get("name") + + # Get role names + roleIds = invitation.get("roleIds", []) + from modules.datamodels.datamodelRbac import Role + for roleId in roleIds: + roleRecords = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roleRecords: + roleLabels.append(roleRecords[0].get("roleLabel", roleId)) + return InvitationValidation( valid=True, reason=None, - mandateId=invitation.get("mandateId"), + mandateId=mandateId, + mandateName=mandateName, featureInstanceId=invitation.get("featureInstanceId"), - roleIds=invitation.get("roleIds", []) + roleIds=roleIds, + roleLabels=roleLabels, + targetUsername=targetUsername ) except Exception as e: @@ -513,6 +642,17 @@ async def accept_invitation( detail="Invitation has reached maximum uses" ) + # Validate username matches - the invitation is bound to a specific user + targetUsername = invitation.get("targetUsername") + if targetUsername and currentUser.username != targetUsername: + logger.warning( + f"User {currentUser.username} tried to accept invitation meant for {targetUsername}" + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Diese Einladung ist für Benutzer '{targetUsername}' bestimmt" + ) + mandateId = invitation.get("mandateId") roleIds = invitation.get("roleIds", []) featureInstanceId = invitation.get("featureInstanceId") diff --git a/modules/routes/routeNotifications.py b/modules/routes/routeNotifications.py new file mode 100644 index 00000000..2016a745 --- /dev/null +++ b/modules/routes/routeNotifications.py @@ -0,0 +1,575 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Notification routes for in-app notifications. +Provides user-specific notification inbox with support for actionable notifications. +""" + +from fastapi import APIRouter, HTTPException, Depends, Request +from typing import List, Dict, Any, Optional +from fastapi import status +import logging +from pydantic import BaseModel, Field + +from modules.auth import limiter, getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelNotification import ( + UserNotification, + NotificationType, + NotificationStatus, + NotificationAction +) +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/notifications", + tags=["Notifications"], + responses={404: {"description": "Not found"}} +) + + +# ============================================================================= +# Request/Response Models +# ============================================================================= + +class NotificationActionRequest(BaseModel): + """Request model for executing a notification action""" + actionId: str = Field(..., description="ID of the action to execute (e.g., 'accept', 'decline')") + + +class UnreadCountResponse(BaseModel): + """Response model for unread count""" + count: int + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def _createNotification( + userId: str, + notificationType: NotificationType, + title: str, + message: str, + referenceType: Optional[str] = None, + referenceId: Optional[str] = None, + actions: Optional[List[NotificationAction]] = None, + icon: Optional[str] = None, + expiresAt: Optional[float] = None +) -> UserNotification: + """ + Create a notification for a user. + This is a helper function that can be imported by other modules. + """ + rootInterface = getRootInterface() + + notification = UserNotification( + userId=userId, + type=notificationType, + title=title, + message=message, + referenceType=referenceType, + referenceId=referenceId, + actions=actions, + icon=icon, + expiresAt=expiresAt + ) + + # Store in database + rootInterface.db.recordCreate( + model_class=UserNotification, + record=notification.model_dump() + ) + + logger.info(f"Created notification {notification.id} for user {userId}: {title}") + return notification + + +def createInvitationNotification( + userId: str, + invitationId: str, + mandateName: str, + inviterName: str +) -> UserNotification: + """ + Create a notification for a pending invitation. + Called when an invitation is created for an existing user. + """ + return _createNotification( + userId=userId, + notificationType=NotificationType.INVITATION, + title="Neue Einladung", + message=f"{inviterName} hat Sie zu '{mandateName}' eingeladen.", + referenceType="Invitation", + referenceId=invitationId, + icon="mail", + actions=[ + NotificationAction(actionId="accept", label="Annehmen", style="primary"), + NotificationAction(actionId="decline", label="Ablehnen", style="danger") + ] + ) + + +# ============================================================================= +# API Endpoints +# ============================================================================= + +@router.get("", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getNotifications( + request: Request, + currentUser: User = Depends(getCurrentUser), + status: Optional[str] = None, + type: Optional[str] = None, + limit: int = 50 +) -> List[Dict[str, Any]]: + """ + Get all notifications for the current user. + + Optionally filter by status (unread, read, actioned, dismissed) or type. + """ + try: + rootInterface = getRootInterface() + + # Build filter + recordFilter = {"userId": str(currentUser.id)} + if status: + recordFilter["status"] = status + if type: + recordFilter["type"] = type + + # Get notifications + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter=recordFilter + ) + + # Sort by creation date (newest first) and limit + notifications = sorted(notifications, key=lambda x: x.get("createdAt", 0), reverse=True) + if limit: + notifications = notifications[:limit] + + return notifications + + except Exception as e: + logger.error(f"Error getting notifications: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to get notifications: {str(e)}" + ) + + +@router.get("/unread-count", response_model=UnreadCountResponse) +@limiter.limit("120/minute") +async def getUnreadCount( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> UnreadCountResponse: + """ + Get the count of unread notifications for the current user. + Used for the notification badge in the header. + """ + try: + rootInterface = getRootInterface() + + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter={ + "userId": str(currentUser.id), + "status": NotificationStatus.UNREAD.value + } + ) + + return UnreadCountResponse(count=len(notifications)) + + except Exception as e: + logger.error(f"Error getting unread count: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to get unread count: {str(e)}" + ) + + +@router.put("/{notificationId}/read", response_model=Dict[str, Any]) +@limiter.limit("60/minute") +async def markAsRead( + request: Request, + notificationId: str, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Mark a notification as read. + """ + try: + rootInterface = getRootInterface() + + # Get the notification + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter={"id": notificationId} + ) + + if not notifications: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Notification not found" + ) + + notification = notifications[0] + + # Verify ownership + if notification.get("userId") != currentUser.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Not authorized to access this notification" + ) + + # Update status + rootInterface.db.recordModify( + model_class=UserNotification, + recordId=notificationId, + record={ + "status": NotificationStatus.READ.value, + "readAt": getUtcTimestamp() + } + ) + + return {"message": "Notification marked as read", "id": notificationId} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error marking notification as read: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to mark notification as read: {str(e)}" + ) + + +@router.put("/mark-all-read", response_model=Dict[str, Any]) +@limiter.limit("10/minute") +async def markAllAsRead( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Mark all notifications as read for the current user. + """ + try: + rootInterface = getRootInterface() + + # Get all unread notifications + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter={ + "userId": currentUser.id, + "status": NotificationStatus.UNREAD.value + } + ) + + currentTime = getUtcTimestamp() + updatedCount = 0 + + for notification in notifications: + rootInterface.db.recordModify( + model_class=UserNotification, + recordId=notification.get("id"), + record={ + "status": NotificationStatus.READ.value, + "readAt": currentTime + } + ) + updatedCount += 1 + + return {"message": f"Marked {updatedCount} notifications as read", "count": updatedCount} + + except Exception as e: + logger.error(f"Error marking all notifications as read: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to mark notifications as read: {str(e)}" + ) + + +@router.post("/{notificationId}/action", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def executeAction( + request: Request, + notificationId: str, + actionRequest: NotificationActionRequest, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Execute an action on a notification (e.g., accept/decline invitation). + """ + try: + rootInterface = getRootInterface() + + # Get the notification + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter={"id": notificationId} + ) + + if not notifications: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Notification not found" + ) + + notification = notifications[0] + + # Verify ownership + if notification.get("userId") != currentUser.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Not authorized to access this notification" + ) + + # Check if already actioned + if notification.get("status") == NotificationStatus.ACTIONED.value: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Notification has already been actioned" + ) + + # Validate action exists + actions = notification.get("actions", []) + validActionIds = [a.get("actionId") if isinstance(a, dict) else a.actionId for a in (actions or [])] + + if actionRequest.actionId not in validActionIds: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid action. Valid actions: {validActionIds}" + ) + + # Execute action based on notification type + actionResult = None + + if notification.get("type") == NotificationType.INVITATION.value: + actionResult = await _handleInvitationAction( + notification=notification, + actionId=actionRequest.actionId, + currentUser=currentUser, + rootInterface=rootInterface + ) + else: + # Generic action handling + actionResult = f"Action '{actionRequest.actionId}' executed" + + # Update notification status + rootInterface.db.recordModify( + model_class=UserNotification, + recordId=notificationId, + record={ + "status": NotificationStatus.ACTIONED.value, + "actionTaken": actionRequest.actionId, + "actionResult": actionResult, + "actionedAt": getUtcTimestamp() + } + ) + + return { + "message": actionResult, + "action": actionRequest.actionId, + "notificationId": notificationId + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error executing notification action: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to execute action: {str(e)}" + ) + + +async def _handleInvitationAction( + notification: Dict[str, Any], + actionId: str, + currentUser: User, + rootInterface +) -> str: + """Handle accept/decline actions for invitation notifications.""" + from modules.datamodels.datamodelInvitation import Invitation + from modules.datamodels.datamodelUam import Mandate + from modules.datamodels.datamodelMembership import UserMandate + + invitationId = notification.get("referenceId") + if not invitationId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No invitation reference found" + ) + + # Get the invitation + invitations = rootInterface.db.getRecordset( + model_class=Invitation, + recordFilter={"id": invitationId} + ) + + if not invitations: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Invitation not found" + ) + + invitation = invitations[0] + + # Verify username matches + if invitation.get("targetUsername") != currentUser.username: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This invitation is for a different user" + ) + + # Check if invitation is still valid + currentTime = getUtcTimestamp() + if invitation.get("expiresAt", 0) < currentTime: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has expired" + ) + + if invitation.get("revokedAt"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has been revoked" + ) + + if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invitation has reached maximum uses" + ) + + if actionId == "accept": + # Accept the invitation - assign roles and mandate access + mandateId = invitation.get("mandateId") + roleIds = invitation.get("roleIds", []) + + # Get mandate name for result message + mandates = rootInterface.db.getRecordset( + model_class=Mandate, + recordFilter={"id": mandateId} + ) + mandateName = mandates[0].get("mandateLabel", mandateId) if mandates else mandateId + + # Check if user already has this mandate + existingMemberships = rootInterface.db.getRecordset( + model_class=UserMandate, + recordFilter={ + "userId": currentUser.id, + "mandateId": mandateId + } + ) + + if existingMemberships: + # Update existing membership with new roles + existingMembership = existingMemberships[0] + existingRoles = existingMembership.get("roleIds", []) + mergedRoles = list(set(existingRoles + roleIds)) + + rootInterface.db.recordModify( + model_class=UserMandate, + recordId=existingMembership.get("id"), + record={"roleIds": mergedRoles} + ) + logger.info(f"Updated UserMandate for user {currentUser.id} in mandate {mandateId}") + else: + # Create new user-mandate relationship + userMandate = UserMandate( + userId=currentUser.id, + mandateId=mandateId, + roleIds=roleIds + ) + rootInterface.db.recordCreate( + model_class=UserMandate, + record=userMandate.model_dump() + ) + logger.info(f"Created UserMandate for user {currentUser.id} in mandate {mandateId}") + + # Mark invitation as used + rootInterface.db.recordModify( + model_class=Invitation, + recordId=invitationId, + record={ + "usedBy": currentUser.id, + "usedAt": currentTime, + "currentUses": invitation.get("currentUses", 0) + 1 + } + ) + + logger.info(f"User {currentUser.id} accepted invitation {invitationId} for mandate {mandateId}") + return f"Einladung angenommen. Sie haben jetzt Zugang zu '{mandateName}'." + + elif actionId == "decline": + # Decline the invitation + # We don't revoke it, just mark the notification as declined + logger.info(f"User {currentUser.id} declined invitation {invitationId}") + return "Einladung abgelehnt." + + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown action: {actionId}" + ) + + +@router.delete("/{notificationId}", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def deleteNotification( + request: Request, + notificationId: str, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Delete/dismiss a notification. + """ + try: + rootInterface = getRootInterface() + + # Get the notification + notifications = rootInterface.db.getRecordset( + model_class=UserNotification, + recordFilter={"id": notificationId} + ) + + if not notifications: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Notification not found" + ) + + notification = notifications[0] + + # Verify ownership + if notification.get("userId") != currentUser.id: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Not authorized to delete this notification" + ) + + # Mark as dismissed (soft delete) + rootInterface.db.recordModify( + model_class=UserNotification, + recordId=notificationId, + record={ + "status": NotificationStatus.DISMISSED.value + } + ) + + return {"message": "Notification dismissed", "id": notificationId} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting notification: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to delete notification: {str(e)}" + ) diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 8ab211cf..8f11a9af 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -20,6 +20,7 @@ from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG +from modules.shared.timeUtils import getUtcTimestamp # Configure logger logger = logging.getLogger(__name__) @@ -322,6 +323,52 @@ Falls Sie sich nicht registriert haben, können Sie diese E-Mail ignorieren.""" logger.error(f"Error sending registration email: {str(emailErr)}") # Don't fail registration if email fails - user can request reset later + # Check for pending invitations and create notifications + try: + from modules.datamodels.datamodelInvitation import Invitation + from modules.routes.routeNotifications import createInvitationNotification + from modules.datamodels.datamodelUam import Mandate + + currentTime = getUtcTimestamp() + pendingInvitations = appInterface.db.getRecordset( + model_class=Invitation, + recordFilter={"targetUsername": userData.username} + ) + + for invitation in pendingInvitations: + # Skip expired, revoked, or fully used invitations + if invitation.get("expiresAt", 0) < currentTime: + continue + if invitation.get("revokedAt"): + continue + if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1): + continue + + # Get mandate name for notification + mandateId = invitation.get("mandateId") + mandateRecords = appInterface.db.getRecordset( + Mandate, + recordFilter={"id": mandateId} + ) + mandateName = mandateRecords[0].get("mandateLabel", "PowerOn") if mandateRecords else "PowerOn" + + # Get inviter name + inviterId = invitation.get("createdBy") + inviter = appInterface.getUserById(inviterId) if inviterId else None + inviterName = (inviter.fullName or inviter.username) if inviter else "PowerOn" + + createInvitationNotification( + userId=str(user.id), + invitationId=str(invitation.get("id")), + mandateName=mandateName, + inviterName=inviterName + ) + logger.info(f"Created notification for new user {userData.username} for invitation {invitation.get('id')}") + + except Exception as notifErr: + logger.warning(f"Failed to create notifications for pending invitations: {notifErr}") + # Don't fail registration if notification creation fails + return { "message": "Registrierung erfolgreich! Bitte prüfen Sie Ihre E-Mail für den Link zum Setzen Ihres Passworts." } diff --git a/modules/system/registry.py b/modules/system/registry.py index 5431b706..8477b045 100644 --- a/modules/system/registry.py +++ b/modules/system/registry.py @@ -111,7 +111,7 @@ def loadFeatureMainModules() -> Dict[str, Any]: def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: """ Register all features' RBAC objects in the catalog. - Also registers system-level RBAC objects. + Also registers system-level RBAC objects and feature definitions. """ results = {} @@ -132,6 +132,20 @@ def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: mainModules = loadFeatureMainModules() for featureName, module in mainModules.items(): + # Register feature definition in catalog (for /api/features/ endpoint) + if hasattr(module, "getFeatureDefinition"): + try: + featureDef = module.getFeatureDefinition() + catalogService.registerFeatureDefinition( + featureCode=featureDef.get("code", featureName), + label=featureDef.get("label", {"en": featureName, "de": featureName}), + icon=featureDef.get("icon", "mdi-puzzle") + ) + logger.info(f"Registered feature definition: {featureDef.get('code', featureName)}") + except Exception as e: + logger.error(f"Error registering feature definition for {featureName}: {e}") + + # Register RBAC objects (UI, RESOURCE, DATA) if hasattr(module, "registerFeature"): try: success = module.registerFeature(catalogService) From 829711f7551b10ab7c95bc0b9cbf994f52655908 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 12:39:00 +0100 Subject: [PATCH 27/32] fixed system and dynamic data rbac --- modules/datamodels/datamodelChat.py | 41 +- .../interfaceFeatureNeutralizer.py | 84 +++- .../mainServiceNeutralization.py | 7 +- modules/interfaces/interfaceBootstrap.py | 387 ++++++++++++++---- modules/interfaces/interfaceDbApp.py | 8 +- modules/interfaces/interfaceDbChat.py | 80 ++-- modules/interfaces/interfaceRbac.py | 101 ++++- modules/routes/routeAdminRbacRules.py | 29 +- modules/routes/routeDataConnections.py | 9 +- modules/routes/routeNotifications.py | 12 + modules/security/rbac.py | 64 +-- modules/system/mainSystem.py | 72 ++-- .../actions/getExpensesFromPdf.py | 46 ++- tests/unit/rbac/test_rbac_bootstrap.py | 10 +- tests/unit/rbac/test_rbac_permissions.py | 26 +- 15 files changed, 695 insertions(+), 281 deletions(-) diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 328bee22..3d71bf63 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -11,15 +11,10 @@ import uuid class ChatStat(BaseModel): + """Statistics for chat operations. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: Optional[str] = Field( - default="", description="ID of the mandate this stat belongs to" - ) - featureInstanceId: Optional[str] = Field( - default="", description="ID of the feature instance this stat belongs to" - ) workflowId: Optional[str] = Field( None, description="Foreign key to workflow (for workflow stats)" ) @@ -39,8 +34,6 @@ registerModelLabels( {"en": "Chat Statistics", "fr": "Statistiques de chat"}, { "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, @@ -54,15 +47,10 @@ registerModelLabels( class ChatLog(BaseModel): + """Log entries for chat workflows. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: Optional[str] = Field( - default="", description="ID of the mandate this log belongs to" - ) - featureInstanceId: Optional[str] = Field( - default="", description="ID of the feature instance this log belongs to" - ) workflowId: str = Field(description="Foreign key to workflow") message: str = Field(description="Log message") type: str = Field(description="Log type (info, warning, error, etc.)") @@ -93,8 +81,6 @@ registerModelLabels( {"en": "Chat Log", "fr": "Journal de chat"}, { "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "message": {"en": "Message", "fr": "Message"}, "type": {"en": "Type", "fr": "Type"}, @@ -107,15 +93,10 @@ registerModelLabels( class ChatDocument(BaseModel): + """Documents attached to chat messages. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: Optional[str] = Field( - default="", description="ID of the mandate this document belongs to" - ) - featureInstanceId: Optional[str] = Field( - default="", description="ID of the feature instance this document belongs to" - ) messageId: str = Field(description="Foreign key to message") fileId: str = Field(description="Foreign key to file") fileName: str = Field(description="Name of the file") @@ -134,8 +115,6 @@ registerModelLabels( {"en": "Chat Document", "fr": "Document de chat"}, { "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "messageId": {"en": "Message ID", "fr": "ID du message"}, "fileId": {"en": "File ID", "fr": "ID du fichier"}, "fileName": {"en": "File Name", "fr": "Nom du fichier"}, @@ -221,15 +200,10 @@ registerModelLabels( class ChatMessage(BaseModel): + """Messages in chat workflows. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) - mandateId: Optional[str] = Field( - default="", description="ID of the mandate this message belongs to" - ) - featureInstanceId: Optional[str] = Field( - default="", description="ID of the feature instance this message belongs to" - ) workflowId: str = Field(description="Foreign key to workflow") parentMessageId: Optional[str] = Field( None, description="Parent message ID for threading" @@ -281,8 +255,6 @@ registerModelLabels( {"en": "Chat Message", "fr": "Message de chat"}, { "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, "documents": {"en": "Documents", "fr": "Documents"}, @@ -326,9 +298,8 @@ registerModelLabels( class ChatWorkflow(BaseModel): + """Chat workflow container. User-owned, no mandate context.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - mandateId: Optional[str] = Field(default="", description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - featureInstanceId: Optional[str] = Field(default="", description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, @@ -402,8 +373,6 @@ registerModelLabels( {"en": "Chat Workflow", "fr": "Flux de travail de chat"}, { "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "status": {"en": "Status", "fr": "Statut"}, "name": {"en": "Name", "fr": "Nom"}, "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, diff --git a/modules/features/neutralization/interfaceFeatureNeutralizer.py b/modules/features/neutralization/interfaceFeatureNeutralizer.py index 970f51ff..54533166 100644 --- a/modules/features/neutralization/interfaceFeatureNeutralizer.py +++ b/modules/features/neutralization/interfaceFeatureNeutralizer.py @@ -12,29 +12,76 @@ from modules.features.neutralization.datamodelFeatureNeutralizer import ( DataNeutraliserConfig, DataNeutralizerAttributes, ) +from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcTimestamp +from modules.datamodels.datamodelUam import User logger = logging.getLogger(__name__) +# Singleton cache for interface instances +_neutralizerInterfaces = {} + class InterfaceFeatureNeutralizer: """Database interface for Neutralizer feature operations""" - def __init__(self, db, currentUser, mandateId: str, userId: str): + # Feature code for RBAC objectKey construction + FEATURE_CODE = "neutralization" + + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """ Initialize the interface with database connection and user context. Args: - db: Database connection instance currentUser: Current user object for RBAC mandateId: Current mandate ID - userId: Current user ID + featureInstanceId: Current feature instance ID """ - self.db = db self.currentUser = currentUser self.mandateId = mandateId - self.userId = userId + self.featureInstanceId = featureInstanceId + self.userId = currentUser.id if currentUser else None + self.db = None + + # Initialize database + self._initializeDatabase() + + def _initializeDatabase(self): + """Initialize the database connection.""" + try: + # Use same database config pattern as other feature interfaces + dbHost = APP_CONFIG.get("DB_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_DATABASE_NEUTRALIZATION", APP_CONFIG.get("DB_DATABASE", "poweron")) + dbUser = APP_CONFIG.get("DB_USER", "postgres") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId, + ) + self.db.initDbSystem() + logger.debug("Neutralizer database initialized successfully") + except Exception as e: + logger.error(f"Error initializing Neutralizer database: {str(e)}") + raise + + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """Sets the user context for the interface.""" + if not currentUser: + logger.info("Initializing interface without user context") + return + + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId def getNeutralizationConfig(self) -> Optional[DataNeutraliserConfig]: """Get the data neutralization configuration for the current user's mandate""" @@ -160,17 +207,34 @@ class InterfaceFeatureNeutralizer: return None -def getInterface(db, currentUser, mandateId: str, userId: str) -> InterfaceFeatureNeutralizer: +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> InterfaceFeatureNeutralizer: """ - Factory function to create a Neutralizer interface instance. + Factory function to get or create a Neutralizer interface instance. + Uses singleton pattern per user context. Args: - db: Database connection currentUser: Current user for RBAC mandateId: Current mandate ID - userId: Current user ID + featureInstanceId: Current feature instance ID Returns: InterfaceFeatureNeutralizer instance """ - return InterfaceFeatureNeutralizer(db, currentUser, mandateId, userId) + global _neutralizerInterfaces + + if not currentUser: + raise ValueError("Valid user context required") + + effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None + + # Include featureInstanceId in cache key for proper isolation + cacheKey = f"{currentUser.id}_{effectiveMandateId}_{effectiveFeatureInstanceId}" + + if cacheKey not in _neutralizerInterfaces: + _neutralizerInterfaces[cacheKey] = InterfaceFeatureNeutralizer(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + else: + # Update user context if needed + _neutralizerInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) + + return _neutralizerInterfaces[cacheKey] diff --git a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py index 4351f400..b4b34cf7 100644 --- a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py @@ -14,7 +14,7 @@ import json from typing import Dict, List, Any, Optional from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes -from modules.features.neutralization.interfaceFeatureNeutralizer import InterfaceFeatureNeutralizer +from modules.features.neutralization.interfaceFeatureNeutralizer import InterfaceFeatureNeutralizer, getInterface as getNeutralizerInterface # Import all necessary classes and functions for neutralization from .subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute @@ -42,11 +42,10 @@ class NeutralizationService: self.interfaceNeutralizer: InterfaceFeatureNeutralizer = None if serviceCenter and serviceCenter.interfaceDbApp: dbApp = serviceCenter.interfaceDbApp - self.interfaceNeutralizer = InterfaceFeatureNeutralizer( - db=dbApp.db, + self.interfaceNeutralizer = getNeutralizerInterface( currentUser=dbApp.currentUser, mandateId=dbApp.mandateId, - userId=dbApp.userId + featureInstanceId=getattr(dbApp, 'featureInstanceId', None) ) # Initialize anonymization processors diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 0e66ce7b..c73c4dd1 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -277,6 +277,9 @@ def initRbacRules(db: DatabaseConnector) -> None: existingRules = db.getRecordset(AccessRule) if existingRules: logger.info(f"RBAC rules already exist ({len(existingRules)} rules)") + # Still ensure UI and DATA rules exist (may have been added later) + _ensureUiContextRules(db) + _ensureDataContextRules(db) return logger.info("Initializing RBAC rules") @@ -377,20 +380,31 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: viewerId = _getRoleId(db, "viewer") # ========================================================================== - # SYSTEM TABLE RULES - Using standardized dot format: data.system.{TableName} + # DATA TABLE RULES - Using semantic namespace structure # ========================================================================== - # All DATA context items MUST use the full objectKey format for consistency. - # This matches the DATA_OBJECTS registration in mainSystem.py. - # Feature tables use: data.feature.{featureCode}.{TableName} + # Namespace structure: + # - data.uam.* → User Access Management (mandantenübergreifend) + # - data.chat.* → Chat/AI-Daten (benutzer-eigen, kein Mandantenkontext) + # - data.files.* → Dateien (benutzer-eigen) + # - data.automation.* → Automation (benutzer-eigen) + # - data.feature.* → Mandanten-/Feature-spezifische Daten (dynamisch) + # + # GROUP-Berechtigung: + # - data.uam.*: GROUP filtert nach Mandant (via UserMandate) + # - data.chat.*, data.files.*, data.automation.*: GROUP = MY (benutzer-eigen) # ========================================================================== + # ------------------------------------------------------------------------- + # UAM Namespace - User Access Management + # ------------------------------------------------------------------------- + # Mandate table - Only SysAdmin (flag) can access, not roles # Regular roles have no access to Mandate table if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="data.system.Mandate", + item="data.uam.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -401,7 +415,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="data.system.Mandate", + item="data.uam.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -412,7 +426,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="data.system.Mandate", + item="data.uam.Mandate", view=False, read=AccessLevel.NONE, create=AccessLevel.NONE, @@ -425,7 +439,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -436,7 +450,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -447,7 +461,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -455,92 +469,37 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # FileItem and UserConnection: All users (user, admin, viewer) only MY-level CRUD - restrictedTables = [ - "data.system.UserConnection", # User connections/sessions - only own records - "data.system.FileItem", # Uploaded files - only own files - ] - - for objectKey in restrictedTables: - # Admin: Only MY-level access (not group-level!) - if adminId: + # UserConnection: All users only MY-level CRUD (UAM namespace) + for roleId in [adminId, userId]: + if roleId: tableRules.append(AccessRule( - roleId=adminId, + roleId=roleId, context=AccessRuleContext.DATA, - item=objectKey, + item="data.uam.UserConnection", view=True, read=AccessLevel.MY, create=AccessLevel.MY, update=AccessLevel.MY, delete=AccessLevel.MY, )) - # User: MY-level CRUD - if userId: - tableRules.append(AccessRule( - roleId=userId, - context=AccessRuleContext.DATA, - item=objectKey, - view=True, - read=AccessLevel.MY, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - # Viewer: MY-level read-only - if viewerId: - tableRules.append(AccessRule( - roleId=viewerId, - context=AccessRuleContext.DATA, - item=objectKey, - view=True, - read=AccessLevel.MY, - create=AccessLevel.NONE, - update=AccessLevel.NONE, - delete=AccessLevel.NONE, - )) - - # Prompt: Special rule - CRUD for MY + Read for GROUP - # Each user can manage own prompts (m) but can read group prompts (g) - if adminId: - tableRules.append(AccessRule( - roleId=adminId, - context=AccessRuleContext.DATA, - item="data.system.Prompt", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) - if userId: - tableRules.append(AccessRule( - roleId=userId, - context=AccessRuleContext.DATA, - item="data.system.Prompt", - view=True, - read=AccessLevel.GROUP, - create=AccessLevel.MY, - update=AccessLevel.MY, - delete=AccessLevel.MY, - )) if viewerId: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="data.system.Prompt", + item="data.uam.UserConnection", view=True, - read=AccessLevel.GROUP, + read=AccessLevel.MY, create=AccessLevel.NONE, update=AccessLevel.NONE, delete=AccessLevel.NONE, )) - # Invitation: Standard group-level access + # Invitation: Standard group-level access (UAM namespace) if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="data.system.Invitation", + item="data.uam.Invitation", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -551,7 +510,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="data.system.Invitation", + item="data.uam.Invitation", view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -562,7 +521,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="data.system.Invitation", + item="data.uam.Invitation", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -570,13 +529,12 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: delete=AccessLevel.NONE, )) - # AuthEvent table - Audit logs (no delete allowed for audit integrity!) - # SysAdmin can delete via isSysAdmin bypass, but regular admins cannot + # AuthEvent table - Audit logs (UAM namespace, no delete for audit integrity!) if adminId: tableRules.append(AccessRule( roleId=adminId, context=AccessRuleContext.DATA, - item="data.system.AuthEvent", + item="data.uam.AuthEvent", view=True, read=AccessLevel.ALL, create=AccessLevel.NONE, @@ -587,7 +545,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=userId, context=AccessRuleContext.DATA, - item="data.system.AuthEvent", + item="data.uam.AuthEvent", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -598,7 +556,120 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None: tableRules.append(AccessRule( roleId=viewerId, context=AccessRuleContext.DATA, - item="data.system.AuthEvent", + item="data.uam.AuthEvent", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # ------------------------------------------------------------------------- + # Chat Namespace - User-owned, no mandate context + # ------------------------------------------------------------------------- + + # Prompt: Only MY-level access (user-owned, no mandate context) + # Each user manages only their own prompts + for roleId in [adminId, userId]: + if roleId: + tableRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.DATA, + item="data.chat.Prompt", + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="data.chat.Prompt", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # ChatWorkflow: Only MY-level access (user-owned, no mandate context) + for roleId in [adminId, userId]: + if roleId: + tableRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.DATA, + item="data.chat.ChatWorkflow", + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="data.chat.ChatWorkflow", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # ------------------------------------------------------------------------- + # Files Namespace - User-owned, no mandate context + # ------------------------------------------------------------------------- + + # FileItem: Only MY-level access (user-owned) + for roleId in [adminId, userId]: + if roleId: + tableRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.DATA, + item="data.files.FileItem", + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="data.files.FileItem", + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # ------------------------------------------------------------------------- + # Automation Namespace - User-owned, no mandate context + # ------------------------------------------------------------------------- + + # AutomationDefinition: Only MY-level access (user-owned) + for roleId in [adminId, userId]: + if roleId: + tableRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.DATA, + item="data.automation.AutomationDefinition", + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + if viewerId: + tableRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item="data.automation.AutomationDefinition", view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -670,6 +741,160 @@ def _createUiContextRules(db: DatabaseConnector) -> None: logger.info(f"Created {len(uiRules)} UI context rules") +def _ensureUiContextRules(db: DatabaseConnector) -> None: + """ + Ensure UI context rules exist for all navigation items. + This is called during bootstrap to add missing UI rules for new navigation items. + + Args: + db: Database connector instance + """ + from modules.system.mainSystem import NAVIGATION_SECTIONS + + adminId = _getRoleId(db, "admin") + userId = _getRoleId(db, "user") + viewerId = _getRoleId(db, "viewer") + + # Get existing UI rules + existingUiRules = db.getRecordset( + AccessRule, + recordFilter={"context": AccessRuleContext.UI.value} + ) + + # Build set of existing (roleId, item) combinations + existingCombinations = set() + for rule in existingUiRules: + roleId = rule.get("roleId") + item = rule.get("item") + if roleId and item: + existingCombinations.add((roleId, item)) + + # Check each navigation item and add missing rules + missingRules = [] + for section in NAVIGATION_SECTIONS: + isAdminSection = section.get("adminOnly", False) + + for item in section.get("items", []): + objectKey = item.get("objectKey") + if not objectKey: + continue + + isAdminOnly = item.get("adminOnly", False) or isAdminSection + + if isAdminOnly: + # Admin-only: only admin role + if adminId and (adminId, objectKey) not in existingCombinations: + missingRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.UI, + item=objectKey, + view=True, + read=None, create=None, update=None, delete=None, + )) + else: + # Public/normal: all roles + for roleId in [adminId, userId, viewerId]: + if roleId and (roleId, objectKey) not in existingCombinations: + missingRules.append(AccessRule( + roleId=roleId, + context=AccessRuleContext.UI, + item=objectKey, + view=True, + read=None, create=None, update=None, delete=None, + )) + + # Create missing rules + if missingRules: + for rule in missingRules: + db.recordCreate(AccessRule, rule) + logger.info(f"Created {len(missingRules)} missing UI context rules") + else: + logger.debug("All UI context rules already exist") + + +def _ensureDataContextRules(db: DatabaseConnector) -> None: + """ + Ensure DATA context rules exist for key tables like ChatWorkflow and AutomationDefinition. + This is called during bootstrap to add missing DATA rules for new tables. + + Args: + db: Database connector instance + """ + adminId = _getRoleId(db, "admin") + userId = _getRoleId(db, "user") + viewerId = _getRoleId(db, "viewer") + + # Get existing DATA rules + existingDataRules = db.getRecordset( + AccessRule, + recordFilter={"context": AccessRuleContext.DATA.value} + ) + + # Build set of existing (roleId, item) combinations + existingCombinations = set() + for rule in existingDataRules: + roleId = rule.get("roleId") + item = rule.get("item") + if roleId and item: + existingCombinations.add((roleId, item)) + + # Define tables that need rules (user-owned, no mandate context) + # Users can only manage their own records (MY-level access) + tablesNeedingRules = [ + "data.chat.ChatWorkflow", + "data.automation.AutomationDefinition", + ] + + missingRules = [] + for objectKey in tablesNeedingRules: + # Admin: MY-level access (user-owned, no mandate context) + if adminId and (adminId, objectKey) not in existingCombinations: + missingRules.append(AccessRule( + roleId=adminId, + context=AccessRuleContext.DATA, + item=objectKey, + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + + # User: MY-level access (user-owned, no mandate context) + if userId and (userId, objectKey) not in existingCombinations: + missingRules.append(AccessRule( + roleId=userId, + context=AccessRuleContext.DATA, + item=objectKey, + view=True, + read=AccessLevel.MY, + create=AccessLevel.MY, + update=AccessLevel.MY, + delete=AccessLevel.MY, + )) + + # Viewer: MY read-only (user-owned, no mandate context) + if viewerId and (viewerId, objectKey) not in existingCombinations: + missingRules.append(AccessRule( + roleId=viewerId, + context=AccessRuleContext.DATA, + item=objectKey, + view=True, + read=AccessLevel.MY, + create=AccessLevel.NONE, + update=AccessLevel.NONE, + delete=AccessLevel.NONE, + )) + + # Create missing rules + if missingRules: + for rule in missingRules: + db.recordCreate(AccessRule, rule) + logger.info(f"Created {len(missingRules)} missing DATA context rules") + else: + logger.debug("All DATA context rules already exist") + + def _createResourceContextRules(db: DatabaseConnector) -> None: """ Create RESOURCE context rules for controlling resource access. diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index a7dfc689..250b2a38 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -1217,10 +1217,10 @@ class AppObjects: The created UserConnection object """ try: - # Get the user - user = self.getUser(userId) - if not user: - raise ValueError(f"User not found: {userId}") + # Note: User verification is skipped here because: + # 1. The caller (route) already has an authenticated currentUser + # 2. Users should always be able to create connections for themselves + # 3. getUser() uses RBAC filtering which may fail for users without UserInDB view permissions # Create new connection with all required fields connection = UserConnection( diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index 3c4d35ad..6a43599b 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -364,10 +364,13 @@ class ChatObjects: return False tableName = modelClass.__name__ + # Use buildDataObjectKey for semantic namespace lookup + from modules.interfaces.interfaceRbac import buildDataObjectKey + objectKey = buildDataObjectKey(tableName) permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, - tableName, + objectKey, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId ) @@ -680,8 +683,7 @@ class ChatObjects: startedAt=workflow.get("startedAt", getUtcTimestamp()), logs=logs, messages=messages, - stats=stats, - mandateId=workflow.get("mandateId", self.mandateId) + stats=stats ) except Exception as e: logger.error(f"Error validating workflow data: {str(e)}") @@ -702,9 +704,22 @@ class ChatObjects: # Set mandateId and featureInstanceId from context for proper data isolation if "mandateId" not in workflowData or not workflowData["mandateId"]: - workflowData["mandateId"] = self.mandateId - if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: - workflowData["featureInstanceId"] = self.featureInstanceId + # Use request context mandateId, or fall back to Root mandate + effectiveMandateId = self.mandateId + if not effectiveMandateId: + # Fall back to Root mandate (first mandate in system) + try: + from modules.datamodels.datamodelUam import Mandate + from modules.security.rootAccess import getRootDbAppConnector + dbAppConn = getRootDbAppConnector() + allMandates = dbAppConn.getRecordset(Mandate) + if allMandates: + effectiveMandateId = allMandates[0].get("id") + logger.debug(f"createWorkflow: Using Root mandate {effectiveMandateId}") + except Exception as e: + logger.warning(f"Could not get Root mandate: {e}") + # Note: Chat data is user-owned, no mandate/featureInstance context stored + # mandateId/featureInstanceId removed from ChatWorkflow model # Use generic field separation based on ChatWorkflow model simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) @@ -714,6 +729,7 @@ class ChatObjects: # Convert to ChatWorkflow model (empty related data for new workflow) + # Note: Chat data is user-owned, no mandate/featureInstance fields return ChatWorkflow( id=created["id"], status=created.get("status", "running"), @@ -728,7 +744,6 @@ class ChatObjects: logs=[], messages=[], stats=[], - mandateId=created.get("mandateId", self.mandateId), workflowMode=created["workflowMode"], maxSteps=created.get("maxSteps", 1) ) @@ -774,8 +789,7 @@ class ChatObjects: startedAt=updated.get("startedAt", workflow.startedAt), logs=logs, messages=messages, - stats=stats, - mandateId=updated.get("mandateId", workflow.mandateId) + stats=stats ) def deleteWorkflow(self, workflowId: str) -> bool: @@ -886,7 +900,7 @@ class ChatObjects: # Apply default sorting by publishedAt if no sort specified if pagination is None or not pagination.sort: - messageDicts.sort(key=lambda x: x.get("publishedAt", getUtcTimestamp())) + messageDicts.sort(key=lambda x: x.get("publishedAt") or getUtcTimestamp()) # Apply filtering (if filters provided) if pagination and pagination.filters: @@ -1026,11 +1040,8 @@ class ChatObjects: if "actionNumber" not in messageData: messageData["actionNumber"] = workflow.currentAction - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in messageData or not messageData["mandateId"]: - messageData["mandateId"] = self.mandateId - if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: - messageData["featureInstanceId"] = self.featureInstanceId + # Note: Chat data is user-owned, no mandate/featureInstance context stored + # mandateId/featureInstanceId removed from ChatMessage model # Use generic field separation based on ChatMessage model simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) @@ -1306,11 +1317,8 @@ class ChatObjects: def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: """Creates a document for a message in normalized table.""" try: - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in documentData or not documentData["mandateId"]: - documentData["mandateId"] = self.mandateId - if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: - documentData["featureInstanceId"] = self.featureInstanceId + # Note: Chat data is user-owned, no mandate/featureInstance context stored + # mandateId/featureInstanceId removed from ChatDocument model # Validate and normalize document data to dict document = ChatDocument(**documentData) @@ -1431,11 +1439,8 @@ class ChatObjects: if "timestamp" not in logData: logData["timestamp"] = getUtcTimestamp() - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in logData or not logData["mandateId"]: - logData["mandateId"] = self.mandateId - if "featureInstanceId" not in logData or not logData["featureInstanceId"]: - logData["featureInstanceId"] = self.featureInstanceId + # Note: Chat data is user-owned, no mandate/featureInstance context stored + # mandateId/featureInstanceId removed from ChatLog model # Add status information if not present if "status" not in logData and "type" in logData: @@ -1500,11 +1505,8 @@ class ChatObjects: if "workflowId" not in statData: raise ValueError("workflowId is required in statData") - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in statData or not statData["mandateId"]: - statData["mandateId"] = self.mandateId - if "featureInstanceId" not in statData or not statData["featureInstanceId"]: - statData["featureInstanceId"] = self.featureInstanceId + # Note: Chat data is user-owned, no mandate/featureInstance context stored + # mandateId/featureInstanceId removed from ChatStat model # Validate the stat data against ChatStat model stat = ChatStat(**statData) @@ -1783,8 +1785,22 @@ class ChatObjects: automationData["id"] = str(uuid.uuid4()) # Ensure mandateId and featureInstanceId are set for proper data isolation - if "mandateId" not in automationData: - automationData["mandateId"] = self.mandateId + if "mandateId" not in automationData or not automationData.get("mandateId"): + # Use request context mandateId, or fall back to Root mandate + effectiveMandateId = self.mandateId + if not effectiveMandateId: + # Fall back to Root mandate (first mandate in system) + try: + from modules.datamodels.datamodelUam import Mandate + from modules.security.rootAccess import getRootDbAppConnector + dbAppConn = getRootDbAppConnector() + allMandates = dbAppConn.getRecordset(Mandate) + if allMandates: + effectiveMandateId = allMandates[0].get("id") + logger.debug(f"createAutomationDefinition: Using Root mandate {effectiveMandateId}") + except Exception as e: + logger.warning(f"Could not get Root mandate: {e}") + automationData["mandateId"] = effectiveMandateId if "featureInstanceId" not in automationData: automationData["featureInstanceId"] = self.featureInstanceId diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index aec97b5a..3e062048 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -7,6 +7,18 @@ Provides RBAC filtering for database queries without connectors importing securi Multi-Tenant Design: - mandateId kommt aus Request-Context (X-Mandate-Id Header) - GROUP-Filter verwendet expliziten mandateId Parameter + +Data Namespace Structure: +- data.uam.{Table} → User Access Management (mandantenübergreifend) +- data.chat.{Table} → Chat/AI-Daten (benutzer-eigen, kein Mandantenkontext) +- data.files.{Table} → Dateien (benutzer-eigen) +- data.automation.{Table} → Automation (benutzer-eigen) +- data.feature.{code}.{Table} → Mandanten-/Feature-spezifische Daten (dynamisch) + +GROUP-Berechtigung: +- data.uam.*: GROUP filtert nach Mandant (via UserMandate) +- data.chat.*, data.files.*, data.automation.*: GROUP = MY (benutzer-eigen, kein Mandantenkontext) +- data.feature.*: GROUP filtert nach mandateId/featureInstanceId """ import logging @@ -21,25 +33,70 @@ from modules.security.rootAccess import getRootDbAppConnector logger = logging.getLogger(__name__) +# ============================================================================= +# Namespace-Mapping für statische Tabellen +# ============================================================================= +# Definiert, welcher Namespace für jede Tabelle verwendet wird. +# Tabellen ohne Eintrag fallen auf "system" zurück (Fallback für Rückwärtskompatibilität). +# ============================================================================= + +TABLE_NAMESPACE = { + # UAM (User Access Management) - mandantenübergreifend + "UserInDB": "uam", + "UserConnection": "uam", + "AuthEvent": "uam", + "Mandate": "uam", + "UserMandate": "uam", + "UserMandateRole": "uam", + "Invitation": "uam", + "Role": "uam", + "AccessRule": "uam", + "FeatureInstance": "uam", + "FeatureAccess": "uam", + "FeatureAccessRole": "uam", + # Chat - benutzer-eigen, kein Mandantenkontext + "ChatWorkflow": "chat", + "ChatMessage": "chat", + "ChatLog": "chat", + "ChatStat": "chat", + "ChatDocument": "chat", + "Prompt": "chat", + # Files - benutzer-eigen + "FileItem": "files", + "FileData": "files", + # Automation - benutzer-eigen + "AutomationDefinition": "automation", +} + +# Namespaces ohne Mandantenkontext - GROUP wird auf MY gemappt +USER_OWNED_NAMESPACES = {"chat", "files", "automation"} + + def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str: """ Build the standardized objectKey for a DATA context item. Format: - - System tables: data.system.{TableName} + - UAM tables: data.uam.{TableName} + - Chat tables: data.chat.{TableName} + - File tables: data.files.{TableName} + - Automation tables: data.automation.{TableName} - Feature tables: data.feature.{featureCode}.{TableName} Args: - tableName: The database table name (e.g., "UserInDB", "TrusteePosition") + tableName: The database table name (e.g., "UserInDB", "ChatWorkflow") featureCode: Optional feature code (e.g., "trustee", "realestate") - If None, assumes system table. + If provided, uses data.feature.{featureCode}.{tableName} Returns: - Full objectKey string (e.g., "data.system.UserInDB" or "data.feature.trustee.TrusteePosition") + Full objectKey string (e.g., "data.uam.UserInDB", "data.chat.ChatWorkflow", + or "data.feature.trustee.TrusteePosition") """ if featureCode: return f"data.feature.{featureCode}.{tableName}" - return f"data.system.{tableName}" + + namespace = TABLE_NAMESPACE.get(tableName, "system") # Fallback für unbekannte Tabellen + return f"data.{namespace}.{tableName}" def getRecordsetWithRBAC( @@ -107,7 +164,7 @@ def getRecordsetWithRBAC( permissions = rbacInstance.getUserPermissions( currentUser, AccessRuleContext.DATA, - objectKey, # Use full objectKey (e.g., "data.system.UserInDB") + objectKey, # Use full objectKey (e.g., "data.uam.UserInDB", "data.chat.ChatWorkflow") mandateId=effectiveMandateId, featureInstanceId=featureInstanceId ) @@ -271,10 +328,32 @@ def buildRbacWhereClause( "values": [currentUser.id] } - # Group records - filter by mandateId + # Group records - filter by mandateId or ownership based on namespace if readLevel == AccessLevel.GROUP: + # Determine namespace for this table + namespace = TABLE_NAMESPACE.get(table, "system") + + # For user-owned namespaces (chat, files, automation): + # GROUP has no meaning - these tables have no mandate context + # Simply ignore GROUP (no filtering) + if namespace in USER_OWNED_NAMESPACES: + return None + + # For UAM and other namespaces: GROUP filters by mandate effectiveMandateId = mandateId + if not effectiveMandateId: + # Fall back to Root mandate (first mandate in system) for GROUP access + # This allows system-level tables to be accessed without explicit mandate context + try: + from modules.datamodels.datamodelUam import Mandate + dbApp = getRootDbAppConnector() + allMandates = dbApp.getRecordset(Mandate) + if allMandates: + effectiveMandateId = allMandates[0].get("id") + except Exception as e: + logger.error(f"Error getting Root mandate: {e}") + if not effectiveMandateId: logger.warning(f"User {currentUser.id} has no mandateId for GROUP access") return {"condition": "1 = 0", "values": []} @@ -324,10 +403,16 @@ def buildRbacWhereClause( logger.error(f"Error building GROUP filter for UserConnection: {e}") return {"condition": "1 = 0", "values": []} + # For system tables without mandateId column (Mandate, Role, etc.): + # No row-level filtering - GROUP access = ALL access for these + elif table in ("Mandate", "Role"): + return None + # For other tables, filter by mandateId field + # Also include records with NULL mandateId for backwards compatibility else: return { - "condition": '"mandateId" = %s', + "condition": '("mandateId" = %s OR "mandateId" IS NULL)', "values": [effectiveMandateId] } diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index d125bc2c..916caf38 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -184,9 +184,12 @@ async def get_all_permissions( recordFilter={"userId": str(reqContext.user.id), "enabled": True} ) + logger.debug(f"UI/RESOURCE permissions: Found {len(userMandates)} UserMandates for user {reqContext.user.id}") + # Collect all role IDs the user has across all mandates for userMandate in userMandates: mandateRoleIds = rootInterface.getRoleIdsForUserMandate(userMandate.get("id")) + logger.debug(f"UI/RESOURCE permissions: UserMandate {userMandate.get('id')} (mandate {userMandate.get('mandateId')}) has {len(mandateRoleIds)} roles: {mandateRoleIds}") for rid in mandateRoleIds: if rid not in roleIds: roleIds.append(rid) @@ -261,20 +264,24 @@ async def get_all_permissions( items.add(rule.item) # For each item, calculate user permissions + # For UI/RESOURCE context: Calculate permissions directly from the collected rules + # (Don't use getUserPermissions with mandateId - that would limit to one mandate's roles) for item in sorted(items): - permissions = interface.rbac.getUserPermissions( - reqContext.user, ctx, item, - mandateId=reqContext.mandateId, - featureInstanceId=reqContext.featureInstanceId - ) + # Find matching rules for this item from the already-collected rules + itemView = False + for rule in allRules[ctx]: + if rule.item == item and rule.view: + itemView = True + break + # Only include if user has view permission - if permissions.view: + if itemView: result[ctx.value.lower()][item] = { - "view": permissions.view, - "read": permissions.read.value if permissions.read else None, - "create": permissions.create.value if permissions.create else None, - "update": permissions.update.value if permissions.update else None, - "delete": permissions.delete.value if permissions.delete else None + "view": True, + "read": None, # UI context doesn't use CRUD permissions + "create": None, + "update": None, + "delete": None } return result diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 37200186..5d84efd9 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -331,13 +331,8 @@ async def create_connection( detail=f"Unsupported connection type: {connection_data.get('type')}" ) - # Get fresh copy of user from database - user = interface.getUser(currentUser.id) - if not user: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="User not found" - ) + # Note: currentUser is already authenticated via JWT - no need to re-verify from database + # The getCurrentUser dependency already validated the user exists # Always create a new connection with PENDING status connection = interface.addUserConnection( diff --git a/modules/routes/routeNotifications.py b/modules/routes/routeNotifications.py index 2016a745..7c8cf9ad 100644 --- a/modules/routes/routeNotifications.py +++ b/modules/routes/routeNotifications.py @@ -19,6 +19,7 @@ from modules.datamodels.datamodelNotification import ( NotificationStatus, NotificationAction ) +from modules.datamodels.datamodelRbac import Role from modules.interfaces.interfaceDbApp import getRootInterface from modules.shared.timeUtils import getUtcTimestamp @@ -452,6 +453,17 @@ async def _handleInvitationAction( mandateId = invitation.get("mandateId") roleIds = invitation.get("roleIds", []) + # Ensure user gets the system "user" role for access to public UI elements (e.g. playground) + userRoles = rootInterface.db.getRecordset( + model_class=Role, + recordFilter={"roleLabel": "user"} + ) + if userRoles: + userRoleId = userRoles[0].get("id") + if userRoleId and userRoleId not in roleIds: + roleIds = roleIds + [userRoleId] + logger.debug(f"Added system 'user' role {userRoleId} to invitation roles") + # Get mandate name for result message mandates = rootInterface.db.getRecordset( model_class=Mandate, diff --git a/modules/security/rbac.py b/modules/security/rbac.py index 34e80105..5d20d1fb 100644 --- a/modules/security/rbac.py +++ b/modules/security/rbac.py @@ -13,7 +13,7 @@ Multi-Tenant Design: import logging from typing import List, Optional, TYPE_CHECKING from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role -from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel +from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel, Mandate from modules.datamodels.datamodelMembership import ( UserMandate, UserMandateRole, @@ -148,6 +148,9 @@ class RbacClass: Get all role IDs for a user in the given context. Uses UserMandate + UserMandateRole for the new multi-tenant model. + Also includes roles from the Root mandate (first mandate) if different + from the requested mandate, so system-level permissions are always available. + Args: user: User object mandateId: Mandate context @@ -156,30 +159,40 @@ class RbacClass: Returns: List of role IDs """ - roleIds = [] - - if not mandateId: - return roleIds + roleIds = set() # Use set to avoid duplicates try: - # Lade UserMandate - userMandates = self.dbApp.getRecordset( - UserMandate, - recordFilter={"userId": user.id, "mandateId": mandateId, "enabled": True} - ) + # Get Root mandate ID (first mandate in system) + allMandates = self.dbApp.getRecordset(Mandate) + rootMandateId = allMandates[0].get("id") if allMandates else None - if not userMandates: - return roleIds + # Collect mandates to check: + # - If mandateId provided: current mandate + Root mandate (if different) + # - If no mandateId: just Root mandate (for system-level access) + mandatesToCheck = [] + if mandateId: + mandatesToCheck.append(mandateId) + if rootMandateId and rootMandateId not in mandatesToCheck: + mandatesToCheck.append(rootMandateId) - userMandateId = userMandates[0].get("id") - - # Lade UserMandateRoles (Mandate-level roles) - userMandateRoles = self.dbApp.getRecordset( - UserMandateRole, - recordFilter={"userMandateId": userMandateId} - ) - - roleIds.extend([r.get("roleId") for r in userMandateRoles if r.get("roleId")]) + # Load roles from each mandate + for checkMandateId in mandatesToCheck: + userMandates = self.dbApp.getRecordset( + UserMandate, + recordFilter={"userId": user.id, "mandateId": checkMandateId, "enabled": True} + ) + + if userMandates: + userMandateId = userMandates[0].get("id") + + # Lade UserMandateRoles (Mandate-level roles) + userMandateRoles = self.dbApp.getRecordset( + UserMandateRole, + recordFilter={"userMandateId": userMandateId} + ) + + foundRoles = [r.get("roleId") for r in userMandateRoles if r.get("roleId")] + roleIds.update(foundRoles) # Load FeatureAccess + FeatureAccessRole (Instance-level roles) if featureInstanceId: @@ -200,12 +213,12 @@ class RbacClass: recordFilter={"featureAccessId": featureAccessId} ) - roleIds.extend([r.get("roleId") for r in featureAccessRoles if r.get("roleId")]) + roleIds.update([r.get("roleId") for r in featureAccessRoles if r.get("roleId")]) except Exception as e: logger.error(f"Error loading role IDs for user {user.id}: {e}") - return roleIds + return list(roleIds) def getRulesForUserBulk( self, @@ -388,7 +401,10 @@ class RbacClass: Example: rule "data.feature.trustee" matches item "data.feature.trustee.TrusteePosition" All items MUST use the full objectKey format: - - System: data.system.{TableName} (e.g., "data.system.UserInDB") + - UAM: data.uam.{TableName} (e.g., "data.uam.UserInDB") + - Chat: data.chat.{TableName} (e.g., "data.chat.ChatWorkflow") + - Files: data.files.{TableName} (e.g., "data.files.FileItem") + - Automation: data.automation.{TableName} (e.g., "data.automation.AutomationDefinition") - Feature: data.feature.{featureCode}.{TableName} (e.g., "data.feature.trustee.TrusteePosition") - UI: ui.{area}.{page} (e.g., "ui.admin.users") diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index 24d1d410..644b121f 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -72,6 +72,7 @@ NAVIGATION_SECTIONS = [ "icon": "FaPlay", "path": "/workflows/playground", "order": 10, + "public": True, }, { "id": "chats", @@ -80,6 +81,7 @@ NAVIGATION_SECTIONS = [ "icon": "FaListAlt", "path": "/workflows/list", "order": 20, + "public": True, }, { "id": "automations", @@ -88,6 +90,7 @@ NAVIGATION_SECTIONS = [ "icon": "FaCogs", "path": "/workflows/automations", "order": 30, + "public": True, }, ], }, @@ -297,72 +300,83 @@ UI_OBJECTS = _buildUiObjectsFromNavigation() # ============================================================================= # System DATA Objects # ============================================================================= +# Namespace structure: +# - data.uam.* → User Access Management (mandantenübergreifend) +# - data.chat.* → Chat/AI-Daten (benutzer-eigen, kein Mandantenkontext) +# - data.files.* → Dateien (benutzer-eigen) +# - data.automation.* → Automation (benutzer-eigen) +# - data.feature.* → Mandanten-/Feature-spezifische Daten (dynamisch) +# ============================================================================= DATA_OBJECTS = [ - # User/Auth tables + # UAM (User Access Management) - mandantenübergreifend { - "objectKey": "data.system.UserInDB", + "objectKey": "data.uam.UserInDB", "label": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, - "meta": {"table": "UserInDB"} + "meta": {"table": "UserInDB", "namespace": "uam"} }, { - "objectKey": "data.system.AuthEvent", + "objectKey": "data.uam.AuthEvent", "label": {"en": "Auth Event", "de": "Auth-Ereignis", "fr": "Événement d'auth"}, - "meta": {"table": "AuthEvent"} + "meta": {"table": "AuthEvent", "namespace": "uam"} }, { - "objectKey": "data.system.UserConnection", + "objectKey": "data.uam.UserConnection", "label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"}, - "meta": {"table": "UserConnection"} + "meta": {"table": "UserConnection", "namespace": "uam"} }, - # Mandate/Membership tables { - "objectKey": "data.system.Mandate", + "objectKey": "data.uam.Mandate", "label": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, - "meta": {"table": "Mandate"} + "meta": {"table": "Mandate", "namespace": "uam"} }, { - "objectKey": "data.system.UserMandate", + "objectKey": "data.uam.UserMandate", "label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"}, - "meta": {"table": "UserMandate"} + "meta": {"table": "UserMandate", "namespace": "uam"} }, { - "objectKey": "data.system.Invitation", + "objectKey": "data.uam.Invitation", "label": {"en": "Invitation", "de": "Einladung", "fr": "Invitation"}, - "meta": {"table": "Invitation"} + "meta": {"table": "Invitation", "namespace": "uam"} }, - # RBAC tables { - "objectKey": "data.system.Role", + "objectKey": "data.uam.Role", "label": {"en": "Role", "de": "Rolle", "fr": "Rôle"}, - "meta": {"table": "Role"} + "meta": {"table": "Role", "namespace": "uam"} }, { - "objectKey": "data.system.AccessRule", + "objectKey": "data.uam.AccessRule", "label": {"en": "Access Rule", "de": "Zugriffsregel", "fr": "Règle d'accès"}, - "meta": {"table": "AccessRule"} + "meta": {"table": "AccessRule", "namespace": "uam"} }, - # Feature tables { - "objectKey": "data.system.FeatureInstance", + "objectKey": "data.uam.FeatureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"}, - "meta": {"table": "FeatureInstance"} + "meta": {"table": "FeatureInstance", "namespace": "uam"} }, - # Content tables + # Chat - benutzer-eigen, kein Mandantenkontext { - "objectKey": "data.system.Prompt", + "objectKey": "data.chat.Prompt", "label": {"en": "Prompt", "de": "Prompt", "fr": "Prompt"}, - "meta": {"table": "Prompt"} + "meta": {"table": "Prompt", "namespace": "chat", "groupDisabled": True} }, { - "objectKey": "data.system.ChatWorkflow", + "objectKey": "data.chat.ChatWorkflow", "label": {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de chat"}, - "meta": {"table": "ChatWorkflow"} + "meta": {"table": "ChatWorkflow", "namespace": "chat", "groupDisabled": True} }, + # Files - benutzer-eigen { - "objectKey": "data.system.FileItem", + "objectKey": "data.files.FileItem", "label": {"en": "File", "de": "Datei", "fr": "Fichier"}, - "meta": {"table": "FileItem"} + "meta": {"table": "FileItem", "namespace": "files", "groupDisabled": True} + }, + # Automation - benutzer-eigen + { + "objectKey": "data.automation.AutomationDefinition", + "label": {"en": "Automation", "de": "Automatisierung", "fr": "Automatisation"}, + "meta": {"table": "AutomationDefinition", "namespace": "automation", "groupDisabled": True} }, ] diff --git a/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py index c2ecb7c9..e583d8bf 100644 --- a/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py +++ b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py @@ -360,7 +360,7 @@ async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, pr from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelDocref import DocumentReferenceList - dbInterface = getDbInterface() + dbInterface = getDbInterface(services.user, mandateId=services.mandateId, featureInstanceId=featureInstanceId) # Create file record fileItem = dbInterface.createFile( @@ -375,40 +375,52 @@ async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, pr logger.info(f"Stored PDF {fileName} ({len(fileContent)} bytes) with fileId: {fileItem.id}") # Step 2: Create ChatDocument referencing the file - # Use workflow context if available - workflowId = services.workflow.id if services.workflow else str(uuid.uuid4()) - messageId = f"expense_import_{workflowId}_{str(uuid.uuid4())[:8]}" - + documentId = str(uuid.uuid4()) chatDocument = ChatDocument( - id=str(uuid.uuid4()), + id=documentId, mandateId=services.mandateId or "", featureInstanceId=featureInstanceId or "", - messageId=messageId, + messageId="", # Will be set when attached to message fileId=fileItem.id, fileName=fileName, fileSize=len(fileContent), mimeType="application/pdf" ) - # Step 3: Create DocumentReferenceList for AI service + # Step 3: Create a proper message with the document attached to the workflow + # This ensures getChatDocumentsFromDocumentList can find the document via workflow.messages + messageData = { + "id": f"msg_expense_import_{str(uuid.uuid4())[:8]}", + "documentsLabel": f"expense_pdf_{fileName}", + "role": "user", + "status": "step", + "message": f"PDF document for expense extraction: {fileName}" + } + + # Use storeMessageWithDocuments to properly create message + document and sync with workflow + createdMessage = services.chat.storeMessageWithDocuments( + services.workflow, + messageData, + [chatDocument.model_dump()] + ) + + # Update documentId to match the created document's actual ID + if createdMessage and createdMessage.documents: + documentId = createdMessage.documents[0].id + + logger.info(f"Created message {createdMessage.id} with ChatDocument {documentId} for AI processing") + + # Step 4: Create DocumentReferenceList for AI service from modules.datamodels.datamodelDocref import DocumentItemReference documentList = DocumentReferenceList( references=[ DocumentItemReference( - documentId=chatDocument.id, + documentId=documentId, fileName=fileName ) ] ) - # Step 4: Store the ChatDocument so AI service can retrieve it - # The AI service uses getChatDocumentsFromDocumentList which queries the database - from modules.interfaces.interfaceDbChat import getInterface as getChatInterface - chatInterface = getChatInterface(services.user, mandateId=services.mandateId, featureInstanceId=featureInstanceId) - chatInterface.createDocument(chatDocument.model_dump()) - - logger.info(f"Created ChatDocument {chatDocument.id} for AI processing") - # Step 5: Call AI with documentList - let AI service handle everything # (extraction, intent analysis, chunking, image processing) options = AiCallOptions( diff --git a/tests/unit/rbac/test_rbac_bootstrap.py b/tests/unit/rbac/test_rbac_bootstrap.py index 05951264..e8b04f07 100644 --- a/tests/unit/rbac/test_rbac_bootstrap.py +++ b/tests/unit/rbac/test_rbac_bootstrap.py @@ -119,9 +119,9 @@ class TestRbacBootstrap: # Should create multiple rules for different tables assert db.recordCreate.call_count > 0 - # Check that Mandate table rules are created with full objectKey + # Check that Mandate table rules are created with full objectKey (UAM namespace) mandateCalls = [call for call in db.recordCreate.call_args_list - if call[0][1].item == "data.system.Mandate"] + if call[0][1].item == "data.uam.Mandate"] assert len(mandateCalls) > 0 # Check that all roles have view=False and no access for Mandate @@ -134,11 +134,11 @@ class TestRbacBootstrap: def testInitRbacRulesSkipsIfExists(self): """Test that initRbacRules skips default rule creation if rules already exist, but adds missing table-specific rules.""" db = Mock() - # Mock existing rules - include rules for ChatWorkflow and Prompt to prevent adding missing rules + # Mock existing rules - include rules for ChatWorkflow and AutomationDefinition to prevent adding missing rules # Need rules for all required roles to fully prevent creation - # Using full objectKey format: data.system.{TableName} + # Using semantic namespace format: data.chat.{TableName}, data.automation.{TableName} existingRules = [] - for table in ["data.system.ChatWorkflow", "data.system.Prompt"]: + for table in ["data.chat.ChatWorkflow", "data.automation.AutomationDefinition"]: for role in ["admin", "user", "viewer"]: existingRules.append({ "id": f"rule_{table}_{role}", diff --git a/tests/unit/rbac/test_rbac_permissions.py b/tests/unit/rbac/test_rbac_permissions.py index a3387f92..b40bebe3 100644 --- a/tests/unit/rbac/test_rbac_permissions.py +++ b/tests/unit/rbac/test_rbac_permissions.py @@ -94,7 +94,7 @@ class TestRbacPermissionResolution: AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB", # Specific rule with full objectKey + item="data.uam.UserInDB", # Specific rule with UAM namespace view=True, read=AccessLevel.MY, create=AccessLevel.NONE, @@ -107,11 +107,11 @@ class TestRbacPermissionResolution: rbac._getRulesForRole = mockGetRulesForRole # Get permissions for UserInDB table - should use specific rule - # Using full objectKey format: data.system.UserInDB + # Using UAM namespace: data.uam.UserInDB permissions = rbac.getUserPermissions( user, AccessRuleContext.DATA, - "data.system.UserInDB" + "data.uam.UserInDB" ) # Most specific rule should win @@ -253,29 +253,29 @@ class TestRbacPermissionResolution: AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB", # Table-level with full objectKey + item="data.uam.UserInDB", # Table-level with UAM namespace view=True, read=AccessLevel.MY ), AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB.email", # Field-level - most specific + item="data.uam.UserInDB.email", # Field-level - most specific view=True, read=AccessLevel.NONE ) ] # Test exact match - rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB.email") + rule = rbac.findMostSpecificRule(rules, "data.uam.UserInDB.email") assert rule is not None - assert rule.item == "data.system.UserInDB.email" + assert rule.item == "data.uam.UserInDB.email" assert rule.read == AccessLevel.NONE # Test table-level match - rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB") + rule = rbac.findMostSpecificRule(rules, "data.uam.UserInDB") assert rule is not None - assert rule.item == "data.system.UserInDB" + assert rule.item == "data.uam.UserInDB" assert rule.read == AccessLevel.MY # Test generic fallback @@ -294,7 +294,7 @@ class TestRbacPermissionResolution: rule1 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.MY, @@ -307,7 +307,7 @@ class TestRbacPermissionResolution: rule2 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.MY, create=AccessLevel.GROUP, # Not allowed @@ -320,7 +320,7 @@ class TestRbacPermissionResolution: rule3 = AccessRule( roleLabel="admin", context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.GROUP, create=AccessLevel.GROUP, @@ -333,7 +333,7 @@ class TestRbacPermissionResolution: rule4 = AccessRule( roleLabel="user", context=AccessRuleContext.DATA, - item="data.system.UserInDB", + item="data.uam.UserInDB", view=True, read=AccessLevel.NONE, create=AccessLevel.MY, # Not allowed without read From 5c4813b10a1e4531d0ab070a34f0bb2d95abf046 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 14:54:47 +0100 Subject: [PATCH 28/32] workflow dynamic tested --- .../services/serviceAi/subStructureFilling.py | 30 ++++++++++++++++++- .../serviceAi/subStructureGeneration.py | 13 ++++++-- .../workflows/processing/workflowProcessor.py | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/modules/services/serviceAi/subStructureFilling.py b/modules/services/serviceAi/subStructureFilling.py index 5145ad54..8f63277f 100644 --- a/modules/services/serviceAi/subStructureFilling.py +++ b/modules/services/serviceAi/subStructureFilling.py @@ -1797,6 +1797,13 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, CHAPTER: {chapterTitle} (Level {chapterLevel}, ID: {chapterId}) GENERATION HINT: {generationHint} +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT sections: Focus on essential information only +- AVOID creating too many sections - combine related content where possible +- Each section should serve a clear purpose with meaningful data +- If no relevant data exists for a topic, do NOT create a section for it +- Prefer ONE comprehensive section over multiple sparse sections + **CRITICAL**: The chapter's generationHint above describes what content this chapter should generate. If the generationHint references documents/images/data, then EACH section that generates content for this chapter MUST assign the relevant ContentParts from AVAILABLE CONTENT PARTS below. NOTE: Chapter already has a heading section. Do NOT generate a heading for the chapter title. @@ -2024,6 +2031,13 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, - Content Type: {contentType} - Generation Hint: {generationHint} +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + ## INSTRUCTIONS 1. Extract all data from the context provided. Do not skip or omit any data. 2. Extract data only from the provided context. Never invent, create, or generate data that is not in the context. @@ -2076,6 +2090,13 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, - Content Type: {contentType} - Generation Hint: {generationHint} +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + ## AVAILABLE CONTENT FOR THIS SECTION {contentPartsText} @@ -2124,13 +2145,20 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, - Content Type: {contentType} - Generation Hint: {generationHint} +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + ## INSTRUCTIONS 1. Generate content based on the Generation Hint above. 2. Create appropriate content that matches the content_type ({contentType}). 3. The content should be relevant to the USER REQUEST and fit the context of surrounding sections. 4. Return only valid JSON with "elements" array. 5. No HTML/styling: Plain text only, no markup. -6. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. +6. Keep content CONCISE - focus on substance, not length. ## OUTPUT FORMAT Return a JSON object with this structure: diff --git a/modules/services/serviceAi/subStructureGeneration.py b/modules/services/serviceAi/subStructureGeneration.py index 64624b84..67b045b3 100644 --- a/modules/services/serviceAi/subStructureGeneration.py +++ b/modules/services/serviceAi/subStructureGeneration.py @@ -420,8 +420,16 @@ CRITICAL RULE: If the user request mentions BOTH: b) Generic content types (article text, main content, body text, etc.) Then chapters that generate those generic content types MUST assign the relevant ContentParts, because the content should relate to or be based on the provided documents/images/data. +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential information only +- AVOID verbose, lengthy, or repetitive text - be concise and direct +- Prioritize FACTS over filler text - no introductions like "In this chapter..." +- Minimize system resources: shorter content = faster processing +- Quality over quantity: precise, meaningful content rather than padding + ## CHAPTER STRUCTURE REQUIREMENTS - Generate chapters based on USER REQUEST - analyze what structure the user wants +- Create ONLY the minimum chapters needed to cover the user's request - avoid over-structuring - IMPORTANT: Each chapter MUST have ALL these fields: - id: Unique identifier (e.g., "chapter_1") - level: Heading level (1, 2, 3, etc.) @@ -431,9 +439,8 @@ Then chapters that generate those generic content types MUST assign the relevant - sections: Empty array [] (REQUIRED - sections are generated in next phase) - contentParts: {{"partId": {{"instruction": "..."}} or {{"caption": "..."}} or both}} - Assign ContentParts as required by CONTENT ASSIGNMENT RULE above - The "instruction" field for each ContentPart MUST contain ALL relevant details from the USER REQUEST that apply to content extraction for this specific chapter. Include all formatting rules, data requirements, constraints, and specifications mentioned in the user request that are relevant for processing this ContentPart in this chapter. -- generationHint: Description of what content to generate for this chapter - The generationHint MUST contain ALL relevant details from the USER REQUEST that apply to this specific chapter. Include all formatting rules, data requirements, constraints, column specifications, validation rules, and any other specifications mentioned in the user request that are relevant for generating content for this chapter. Do NOT use generic descriptions - include specific details from the user request. -- The number of chapters depends on the user request - create only what is requested +- generationHint: Keep CONCISE but include relevant details from the USER REQUEST. Focus on WHAT to generate, not HOW to phrase it verbosely. +- The number of chapters depends on the user request - create only what is requested. Do NOT create chapters for topics without available data. CRITICAL: Only create chapters for CONTENT sections, not for formatting/styling requirements. Formatting/styling requirements to be included in each generationHint if needed. diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 11879e9d..38763f51 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -13,7 +13,7 @@ from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped -from modules.datamodels.datamodelAi import OperationTypeEnum, PriorityEnum, ProcessingModeEnum, AiCallOptions +from modules.datamodels.datamodelAi import OperationTypeEnum, PriorityEnum, ProcessingModeEnum, AiCallOptions, AiCallRequest from modules.shared.jsonUtils import extractJsonString, repairBrokenJson if TYPE_CHECKING: From 97cbda0ef23eda9ef87399ec9b736a29f2197636 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 23:26:30 +0100 Subject: [PATCH 29/32] fixed ai call end to end with saas multimandate --- modules/aicore/aicoreModelSelector.py | 14 +- modules/datamodels/datamodelChat.py | 7 +- .../automation/subAutomationTemplates.py | 2 +- .../trustee/datamodelFeatureTrustee.py | 33 +- .../trustee/interfaceFeatureTrustee.py | 73 ++--- modules/routes/routeChat.py | 6 +- modules/services/serviceAi/mainServiceAi.py | 39 ++- .../services/serviceAi/subStructureFilling.py | 131 ++++---- modules/workflows/automation/mainWorkflow.py | 11 +- .../actions/getExpensesFromPdf.py | 299 +++++++++++------- .../methodSharepoint/helpers/apiClient.py | 12 + 11 files changed, 395 insertions(+), 232 deletions(-) diff --git a/modules/aicore/aicoreModelSelector.py b/modules/aicore/aicoreModelSelector.py index eeda64d9..8bebb2d7 100644 --- a/modules/aicore/aicoreModelSelector.py +++ b/modules/aicore/aicoreModelSelector.py @@ -72,10 +72,16 @@ class ModelSelector: promptSize = len(prompt.encode("utf-8")) contextSize = len(context.encode("utf-8")) totalSize = promptSize + contextSize - # Convert bytes to approximate tokens (1 token ≈ 4 bytes) - promptTokens = promptSize / 4 - contextTokens = contextSize / 4 - totalTokens = totalSize / 4 + # Convert bytes to approximate tokens + # Conservative estimate: 1 token ≈ 2 bytes (for safety margin) + # Note: Actual tokenization varies by content type and model + # - English text: ~4 bytes/token + # - Structured data/JSON: ~2-3 bytes/token + # - Base64/encoded data: ~1.5-2 bytes/token + bytesPerToken = 2 # Conservative estimate for mixed content + promptTokens = promptSize / bytesPerToken + contextTokens = contextSize / bytesPerToken + totalTokens = totalSize / bytesPerToken logger.debug(f"Request sizes - Prompt: {promptTokens:.0f} tokens ({promptSize} bytes), Context: {contextTokens:.0f} tokens ({contextSize} bytes), Total: {totalTokens:.0f} tokens ({totalSize} bytes)") diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 3d71bf63..22c07aa2 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -220,11 +220,12 @@ class ChatMessage(BaseModel): ) role: str = Field(description="Role of the message sender") status: str = Field(description="Status of the message (first, step, last)") - sequenceNr: int = Field( + sequenceNr: Optional[int] = Field( + default=0, description="Sequence number of the message (set automatically)" ) - publishedAt: float = Field( - default_factory=getUtcTimestamp, + publishedAt: Optional[float] = Field( + default=None, description="When the message was published (UTC timestamp in seconds)", ) success: Optional[bool] = Field( diff --git a/modules/features/automation/subAutomationTemplates.py b/modules/features/automation/subAutomationTemplates.py index e95ca04d..420203ec 100644 --- a/modules/features/automation/subAutomationTemplates.py +++ b/modules/features/automation/subAutomationTemplates.py @@ -399,7 +399,7 @@ AUTOMATION_TEMPLATES: Dict[str, Any] = { "connectionName": "", "sharepointFolder": "", "featureInstanceId": "", - "extractionPrompt": "Du bist ein Spezialist für die Extraktion von Spesendaten aus PDF-Dokumenten.\n\nAUFGABE:\nExtrahiere alle Speseneinträge aus dem bereitgestellten PDF-Dokument und gib sie im CSV-Format zurück.\n\nWICHTIGE REGELN:\n1. Pro MwSt-Prozentsatz einen separaten Datensatz erstellen\n2. Alle Datensätze zusammen müssen den Gesamtbetrag des Dokuments ergeben\n3. Der gesamte extrahierte Text des Dokuments muss im Feld \"desc\" erfasst werden\n4. Feld \"company\" enthält den Lieferanten/Verkäufer der Buchung\n5. Tags müssen aus dieser Liste gewählt werden: customer, meeting, license, subscription, fuel, food, material\n - Mehrere zutreffende Tags mit Komma trennen\n\nCSV-SPALTEN (in dieser Reihenfolge):\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n\nDATENFORMAT:\n- valuta: YYYY-MM-DD (Valutadatum)\n- transactionDateTime: Unix-Timestamp in Sekunden (Transaktionszeitpunkt)\n- company: Lieferant/Verkäufer Name\n- desc: Vollständiger extrahierter Text des Dokuments\n- tags: Komma-getrennte Tags aus der erlaubten Liste\n- bookingCurrency: Währungscode (CHF, EUR, USD, GBP)\n- bookingAmount: Buchungsbetrag als Dezimalzahl\n- originalCurrency: Original-Währungscode\n- originalAmount: Original-Betrag als Dezimalzahl\n- vatPercentage: MwSt-Prozentsatz (z.B. 8.1 für 8.1%)\n- vatAmount: MwSt-Betrag als Dezimalzahl\n\nBEISPIEL OUTPUT:\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",food,CHF,45.50,CHF,45.50,2.6,1.15\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",material,CHF,12.30,CHF,12.30,8.1,0.92\n\nHINWEISE:\n- Wenn nur ein MwSt-Satz vorhanden ist, einen Datensatz erstellen\n- Wenn mehrere MwSt-Sätze vorhanden sind (z.B. Lebensmittel 2.6% und Non-Food 8.1%), separate Datensätze erstellen\n- Bei fehlenden Informationen: leeres Feld oder Standardwert\n- Keine Anführungszeichen um numerische Werte" + "extractionPrompt": "Du bist ein Spezialist für die Extraktion von Belegdaten aus PDF-Dokumenten.\n\nAUFGABE:\nExtrahiere die Daten aus dem bereitgestellten Zahlungsbeleg und erstelle EINE EINZIGE CSV-Tabelle mit allen Datensätzen.\n\nOUTPUT-STRUKTUR:\nErstelle genau EINE Tabelle mit den folgenden Spalten. Alle extrahierten Datensätze kommen in diese eine Tabelle als Zeilen.\n\nWICHTIGE REGELN:\n1. Pro MwSt-Prozentsatz einen separaten Datensatz (= Zeile) erstellen\n2. Alle Datensätze zusammen müssen den Gesamtbetrag des Dokuments ergeben\n3. Der gesamte extrahierte Text des Dokuments muss im Feld \"desc\" erfasst werden\n4. Feld \"company\" enthält den Lieferanten/Verkäufer der Buchung\n5. Tags müssen aus dieser Liste gewählt werden: customer, meeting, license, subscription, fuel, food, material\n - Mehrere zutreffende Tags mit Komma trennen\n\nCSV-SPALTEN (in dieser Reihenfolge):\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n\nDATENFORMAT:\n- valuta: YYYY-MM-DD (Valutadatum)\n- transactionDateTime: Unix-Timestamp in Sekunden (Transaktionszeitpunkt)\n- company: Lieferant/Verkäufer Name\n- desc: Vollständiger extrahierter Text des Dokuments\n- tags: Komma-getrennte Tags aus der erlaubten Liste\n- bookingCurrency: Währungscode (CHF, EUR, USD, GBP)\n- bookingAmount: Buchungsbetrag als Dezimalzahl\n- originalCurrency: Original-Währungscode\n- originalAmount: Original-Betrag als Dezimalzahl\n- vatPercentage: MwSt-Prozentsatz (z.B. 8.1 für 8.1%)\n- vatAmount: MwSt-Betrag als Dezimalzahl\n\nHINWEISE:\n- Wenn nur ein MwSt-Satz vorhanden ist, einen Datensatz erstellen\n- Wenn mehrere MwSt-Sätze vorhanden sind (z.B. Lebensmittel 2.6% und Non-Food 8.1%), separate Datensätze erstellen\n- Bei fehlenden Informationen: leeres Feld oder Standardwert" } } ] diff --git a/modules/features/trustee/datamodelFeatureTrustee.py b/modules/features/trustee/datamodelFeatureTrustee.py index 8b13dff1..d729c1e5 100644 --- a/modules/features/trustee/datamodelFeatureTrustee.py +++ b/modules/features/trustee/datamodelFeatureTrustee.py @@ -279,7 +279,10 @@ registerModelLabels( class TrusteeDocument(BaseModel): - """Contains document references and receipts for bookings. + """Contains document references for bookings. + + Documents reference files in the central Files table via fileId. + This allows file content to be stored once and referenced by multiple features. Note: organisationId and contractId removed as per architecture decision: - The feature instance IS the organisation @@ -294,11 +297,11 @@ class TrusteeDocument(BaseModel): "frontend_required": False } ) - documentData: Optional[bytes] = Field( + fileId: Optional[str] = Field( default=None, - description="The file content (binary)", + description="Reference to central Files table (Files.id)", json_schema_extra={ - "frontend_type": "file", + "frontend_type": "file_reference", "frontend_readonly": False, "frontend_required": False } @@ -321,6 +324,24 @@ class TrusteeDocument(BaseModel): "frontend_options": "/api/trustee/mime-types/options" } ) + sourceType: Optional[str] = Field( + default=None, + description="Source type (e.g., 'sharepoint', 'upload', 'email')", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) + sourceLocation: Optional[str] = Field( + default=None, + description="Original source location (e.g., SharePoint path)", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) mandateId: Optional[str] = Field( default=None, description="Mandate ID (auto-set from context)", @@ -349,9 +370,11 @@ registerModelLabels( {"en": "Document", "fr": "Document", "de": "Dokument"}, { "id": {"en": "ID", "fr": "ID", "de": "ID"}, - "documentData": {"en": "Document Data", "fr": "Données du document", "de": "Dokumentdaten"}, + "fileId": {"en": "File Reference", "fr": "Référence du fichier", "de": "Datei-Referenz"}, "documentName": {"en": "Document Name", "fr": "Nom du document", "de": "Dokumentname"}, "documentMimeType": {"en": "MIME Type", "fr": "Type MIME", "de": "MIME-Typ"}, + "sourceType": {"en": "Source Type", "fr": "Type de source", "de": "Quelltyp"}, + "sourceLocation": {"en": "Source Location", "fr": "Emplacement source", "de": "Quellort"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index df8038f9..c3400752 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -177,8 +177,7 @@ class TrusteeObjects: AccessRuleContext.DATA, tableName, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) if not permissions.view: @@ -205,8 +204,7 @@ class TrusteeObjects: AccessRuleContext.DATA, tableName, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) if not permissions.view: @@ -270,8 +268,7 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) logger.debug(f"getAllOrganisations: getRecordsetWithRBAC returned {len(records)} records") @@ -364,8 +361,7 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all roles @@ -475,8 +471,7 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all records @@ -535,8 +530,7 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -553,8 +547,7 @@ class TrusteeObjects: recordFilter={"userId": userId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) # Users with ALL access level (from system RBAC) see all records @@ -671,8 +664,7 @@ class TrusteeObjects: recordFilter=None, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) totalItems = len(records) @@ -705,8 +697,7 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="label", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -780,8 +771,8 @@ class TrusteeObjects: createdRecord = self.db.recordCreate(TrusteeDocument, data) if createdRecord and createdRecord.get("id"): - # Remove binary data and metadata from Pydantic model - cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_") and k != "documentData"} + # Remove metadata from Pydantic model + cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} return TrusteeDocument(**cleanedRecord) return None @@ -795,12 +786,25 @@ class TrusteeObjects: return TrusteeDocument(**cleanedRecord) def getDocumentData(self, documentId: str) -> Optional[bytes]: - """Get document binary data.""" + """Get document binary data via fileId reference to central Files table.""" records = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) record = records[0] if records else None - if record: - return record.get("documentData") - return None + if not record: + return None + + # New model: fileId references central Files table + fileId = record.get("fileId") + if fileId: + from modules.interfaces.interfaceDbManagement import getInterface as getDbInterface + dbInterface = getDbInterface(self.currentUser, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId) + fileData = dbInterface.getFileData(fileId) + if fileData: + return fileData + logger.warning(f"File data not found for fileId {fileId}") + return None + + # Legacy fallback: documentData was stored directly (for migration) + return record.get("documentData") def getAllDocuments(self, params: Optional[PaginationParams] = None) -> PaginatedResult: """Get all documents with RBAC filtering + feature-level access filtering (metadata only).""" @@ -812,8 +816,7 @@ class TrusteeObjects: recordFilter=None, orderBy="documentName", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) # Convert dicts to Pydantic objects (remove binary data and internal fields) @@ -852,8 +855,7 @@ class TrusteeObjects: recordFilter={"contractId": contractId}, orderBy="documentName", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) result = [] @@ -960,8 +962,7 @@ class TrusteeObjects: recordFilter=None, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) # Convert dicts to Pydantic objects (remove internal fields) @@ -1000,8 +1001,7 @@ class TrusteeObjects: recordFilter={"contractId": contractId}, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -1015,8 +1015,7 @@ class TrusteeObjects: recordFilter={"organisationId": organisationId}, orderBy="valuta", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] @@ -1173,8 +1172,7 @@ class TrusteeObjects: recordFilter={"positionId": positionId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] @@ -1188,8 +1186,7 @@ class TrusteeObjects: recordFilter={"documentId": documentId}, orderBy="id", mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode=self.FEATURE_CODE + featureInstanceId=self.featureInstanceId ) return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links] diff --git a/modules/routes/routeChat.py b/modules/routes/routeChat.py index 22aa764e..137b4a99 100644 --- a/modules/routes/routeChat.py +++ b/modules/routes/routeChat.py @@ -53,7 +53,8 @@ async def start_workflow( """ try: # Start or continue workflow using playground controller - workflow = await chatStart(context.user, userInput, workflowMode, workflowId) + mandateId = str(context.mandateId) if context.mandateId else None + workflow = await chatStart(context.user, userInput, workflowMode, workflowId, mandateId=mandateId) return workflow @@ -75,7 +76,8 @@ async def stop_workflow( """Stops a running workflow.""" try: # Stop workflow using playground controller - workflow = await chatStop(context.user, workflowId) + mandateId = str(context.mandateId) if context.mandateId else None + workflow = await chatStop(context.user, workflowId, mandateId=mandateId) return workflow diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 296a8032..a728bafc 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -785,14 +785,39 @@ Respond with ONLY a JSON object in this exact format: if part.data ]) - # Call AI with extracted content - aiRequest = AiCallRequest( - prompt=f"{prompt}\n\nExtracted Content:\n{contentText}", - context="", - options=options - ) + # Check content size and use chunking if needed + # Conservative estimate: 2 bytes per token, 80% of model limit for safety + contentSizeBytes = len(contentText.encode('utf-8')) + promptSizeBytes = len(prompt.encode('utf-8')) + totalSizeBytes = contentSizeBytes + promptSizeBytes + estimatedTokens = totalSizeBytes / 2 # Conservative: 2 bytes per token - aiResponse = await self.callAi(aiRequest) + # Get max model context (use Claude's 200k as reference, 80% = 160k tokens) + maxSafeTokens = 160000 + + if estimatedTokens > maxSafeTokens: + # Content too large - use chunking via ExtractionService + logger.warning(f"Content too large for single AI call: ~{estimatedTokens:.0f} tokens (limit: {maxSafeTokens}). Using chunked processing.") + + # Use ExtractionService for chunked processing + extractionService = self.services.extraction + aiResponse = await extractionService.processContentPartsWithPrompt( + contentParts=contentParts, + prompt=prompt, + aiObjects=self.aiObjects, + options=options, + operationId=extractOperationId, + parentOperationId=parentOperationId + ) + else: + # Content fits - use single AI call + aiRequest = AiCallRequest( + prompt=f"{prompt}\n\nExtracted Content:\n{contentText}", + context="", + options=options + ) + + aiResponse = await self.callAi(aiRequest) # Create response document resultDocument = DocumentData( diff --git a/modules/services/serviceAi/subStructureFilling.py b/modules/services/serviceAi/subStructureFilling.py index 8f63277f..9b503567 100644 --- a/modules/services/serviceAi/subStructureFilling.py +++ b/modules/services/serviceAi/subStructureFilling.py @@ -567,7 +567,8 @@ class StructureFiller: userPrompt: str, all_sections_list: List[Dict[str, Any]], language: str, - calculateOverallProgress: callable + outputFormat: str = "txt", + calculateOverallProgress: callable = None ) -> List[Dict[str, Any]]: """ Process a single section and return its elements. @@ -761,7 +762,8 @@ class StructureFiller: allSections=all_sections_list, sectionIndex=sectionIndex, isAggregation=isAggregation, - language=language + language=language, + outputFormat=outputFormat ) sectionOperationId = f"{fillOperationId}_section_{sectionId}" @@ -949,7 +951,8 @@ class StructureFiller: allSections=all_sections_list, sectionIndex=sectionIndex, isAggregation=False, - language=language + language=language, + outputFormat=outputFormat ) sectionOperationId = f"{fillOperationId}_section_{sectionId}" @@ -1214,7 +1217,8 @@ class StructureFiller: allSections=all_sections_list, sectionIndex=sectionIndex, isAggregation=False, - language=language + language=language, + outputFormat=outputFormat ) sectionOperationId = f"{fillOperationId}_section_{sectionId}" @@ -1540,6 +1544,7 @@ class StructureFiller: for doc in chapterStructure.get("documents", []): docId = doc.get("id", "unknown") docLanguage = self._getDocumentLanguage(chapterStructure, docId) + docFormat = doc.get("outputFormat", "txt") # Get output format for this document for chapter in doc.get("chapters", []): chapterId = chapter.get("id", "unknown") @@ -1555,7 +1560,8 @@ class StructureFiller: "sectionIndex": sectionIndex, "chapterSectionCount": chapterSectionCount, "section": section, - "docLanguage": docLanguage + "docLanguage": docLanguage, + "docFormat": docFormat # Include output format }) logger.info(f"Starting FULLY PARALLEL section generation: {totalSections} sections across {totalChapters} chapters") @@ -1577,6 +1583,7 @@ class StructureFiller: userPrompt=userPrompt, all_sections_list=all_sections_list, language=taskInfo["docLanguage"], + outputFormat=taskInfo.get("docFormat", "txt"), # Pass output format calculateOverallProgress=lambda *args: completedSections[0] / totalSections if totalSections > 0 else 1.0 ) @@ -1826,23 +1833,11 @@ If AVAILABLE CONTENT PARTS are listed above, then EVERY section that generates c - If chapter's generationHint references documents/images/data AND section generates content for that chapter → section MUST assign relevant ContentParts - Empty contentPartIds [] are only allowed if section generates content WITHOUT referencing any available ContentParts AND WITHOUT relating to chapter's generationHint -## CONTENT TYPES -Available content types for sections: table, bullet_list, heading, paragraph, code_block, image +## ACCEPTED CONTENT TYPES FOR THIS FORMAT +The document output format ({outputFormat}) accepts only the following content types: +{', '.join(acceptedSectionTypes)} -## ACCEPTED SECTION TYPES FOR THIS FORMAT -The document output format ({outputFormat}) accepts only the following section types: -{', '.join(acceptedSectionTypes) if acceptedSectionTypes else 'All section types'} - -**IMPORTANT**: Only create sections with content types from the accepted list above. Do not create sections with types that are not accepted by this format. - -## FORMAT-APPROPRIATE SECTION STRUCTURE -When determining which sections to create for this chapter, consider the document's output format ({outputFormat}) and ensure sections are structured appropriately for that format: -- Different formats have different capabilities and constraints -- Structure sections to match what the format can effectively represent -- Consider what content types work best for each format -- Ensure the section structure aligns with the format's strengths and limitations -- Select content types that are well-suited for the target format -- **CRITICAL**: Only use section types from the ACCEPTED SECTION TYPES list above +**CRITICAL**: Only create sections with content types from this list. Other types will fail. useAiCall RULE (simple): - useAiCall: true → Content needs AI processing (extract, transform, generate, filter, summarize) @@ -1853,7 +1848,7 @@ RETURN JSON: "sections": [ {{ "id": "section_1", - "content_type": "paragraph", + "content_type": "{acceptedSectionTypes[0]}", "contentPartIds": ["extracted_part_id"], "generationHint": "Description of what to extract or generate", "useAiCall": true, @@ -1897,7 +1892,8 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. allSections: Optional[List[Dict[str, Any]]] = None, sectionIndex: Optional[int] = None, isAggregation: bool = False, - language: str = "en" + language: str = "en", + outputFormat: str = "txt" ) -> tuple[str, str]: """Baue Prompt für Section-Generierung mit vollständigem Kontext.""" # Filtere None-Werte @@ -2005,14 +2001,29 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. for next in nextSections: contextText += f"- {next['id']} ({next['content_type']}): {next['generation_hint']}\n" - contentStructureExample = self._getContentStructureExample(contentType) + # Get accepted section types for the output format + acceptedTypesAggr = self._getAcceptedSectionTypesForFormat(outputFormat) + + # CRITICAL: If the section's content_type is not supported by the output format, + # use the first accepted type instead. E.g., CSV only supports 'table', so + # even if section says 'code_block', we must output as 'table'. + effectiveContentType = contentType + if contentType not in acceptedTypesAggr and acceptedTypesAggr: + effectiveContentType = acceptedTypesAggr[0] + logger.debug(f"Section {sectionId}: Content type '{contentType}' not supported by format '{outputFormat}', using '{effectiveContentType}' instead") + + contentStructureExample = self._getContentStructureExample(effectiveContentType) + + # Build format note for the prompt - purely dynamic from renderer + # Always show what types are accepted for this format + formatNoteAggr = f"\n- Target Output Format: {outputFormat.upper()} (accepted content types: {', '.join(acceptedTypesAggr)})" # Create template structure explicitly (not extracted from prompt) # This ensures exact identity between initial and continuation prompts templateStructure = f"""{{ "elements": [ {{ - "type": "{contentType}", + "type": "{effectiveContentType}", "content": {contentStructureExample} }} ] @@ -2022,14 +2033,14 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. prompt = f"""# TASK: Generate Section Content (Aggregation) Return only valid JSON. No explanatory text, no comments, no markdown formatting outside JSON. -If ContentParts have no data, return: {{"elements": [{{"type": "{contentType}", "content": {{"headers": [], "rows": []}}}}]}} +If ContentParts have no data, return: {{"elements": [{{"type": "{effectiveContentType}", "content": {{"headers": [], "rows": []}}}}]}} LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. ## SECTION METADATA - Section ID: {sectionId} -- Content Type: {contentType} -- Generation Hint: {generationHint} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} ## CONTENT EFFICIENCY PRINCIPLES - Generate COMPACT content: Focus on essential facts only @@ -2044,7 +2055,7 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, 3. If the context contains no data, return empty structures (empty rows array for tables). 4. Aggregate all data into one element (e.g., one table). 5. For table: Extract all rows from the context. Return {{"headers": [...], "rows": []}} only if no data exists. -6. Format based on content_type ({contentType}). +6. Format based on content_type ({effectiveContentType}). 7. No HTML/styling: Plain text only, no markup. 8. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. @@ -2055,7 +2066,7 @@ Return a JSON object with this structure: {{ "elements": [ {{ - "type": "{contentType}", + "type": "{effectiveContentType}", "content": {contentStructureExample} }} ] @@ -2087,8 +2098,8 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, ## SECTION METADATA - Section ID: {sectionId} -- Content Type: {contentType} -- Generation Hint: {generationHint} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} ## CONTENT EFFICIENCY PRINCIPLES - Generate COMPACT content: Focus on essential facts only @@ -2103,7 +2114,7 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, ## INSTRUCTIONS 1. Extract data only from provided ContentParts. Never invent or generate data. 2. If ContentParts contain no data, return empty structures (empty rows array for tables). -3. Format based on content_type ({contentType}). +3. Format based on content_type ({effectiveContentType}). 4. Return only valid JSON with "elements" array. 5. No HTML/styling: Plain text only, no markup. 6. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. @@ -2114,7 +2125,7 @@ Return a JSON object with this structure: {{ "elements": [ {{ - "type": "{contentType}", + "type": "{effectiveContentType}", "content": {contentStructureExample} }} ] @@ -2142,8 +2153,8 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, ## SECTION METADATA - Section ID: {sectionId} -- Content Type: {contentType} -- Generation Hint: {generationHint} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} ## CONTENT EFFICIENCY PRINCIPLES - Generate COMPACT content: Focus on essential facts only @@ -2154,7 +2165,7 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, ## INSTRUCTIONS 1. Generate content based on the Generation Hint above. -2. Create appropriate content that matches the content_type ({contentType}). +2. Create appropriate content that matches the content_type ({effectiveContentType}). 3. The content should be relevant to the USER REQUEST and fit the context of surrounding sections. 4. Return only valid JSON with "elements" array. 5. No HTML/styling: Plain text only, no markup. @@ -2166,7 +2177,7 @@ Return a JSON object with this structure: {{ "elements": [ {{ - "type": "{contentType}", + "type": "{effectiveContentType}", "content": {contentStructureExample} }} ] @@ -2557,28 +2568,26 @@ CRITICAL: Returns: List of accepted section content types (e.g., ["table", "code_block"]) + + Raises: + ValueError: If renderer not found or doesn't provide accepted types """ - try: - from modules.services.serviceGeneration.renderers.registry import getRenderer - - # Get renderer for this format - renderer = getRenderer(outputFormat, self.services) - - if renderer and hasattr(renderer, 'getAcceptedSectionTypes'): - # Query renderer for accepted types - acceptedTypes = renderer.getAcceptedSectionTypes(outputFormat) - if acceptedTypes: - logger.debug(f"Renderer for format '{outputFormat}' accepts section types: {acceptedTypes}") - return acceptedTypes - - # Fallback: if no renderer or method not found, return all types - from modules.datamodels.datamodelJson import supportedSectionTypes - logger.debug(f"No renderer found for format '{outputFormat}' or method not available, using all section types") - return list(supportedSectionTypes) - - except Exception as e: - logger.warning(f"Error querying renderer for accepted section types for format '{outputFormat}': {str(e)}") - # Fallback: return all types - from modules.datamodels.datamodelJson import supportedSectionTypes - return list(supportedSectionTypes) + from modules.services.serviceGeneration.renderers.registry import getRenderer + + # Get renderer for this format - NO FALLBACK + renderer = getRenderer(outputFormat, self.services) + + if not renderer: + raise ValueError(f"No renderer found for output format '{outputFormat}'. Check renderer registry.") + + if not hasattr(renderer, 'getAcceptedSectionTypes'): + raise ValueError(f"Renderer for '{outputFormat}' does not implement getAcceptedSectionTypes(). Add this method to the renderer.") + + acceptedTypes = renderer.getAcceptedSectionTypes(outputFormat) + + if not acceptedTypes: + raise ValueError(f"Renderer for '{outputFormat}' returned empty accepted types. Fix getAcceptedSectionTypes() in the renderer.") + + logger.debug(f"Renderer for '{outputFormat}' accepts: {acceptedTypes}") + return acceptedTypes diff --git a/modules/workflows/automation/mainWorkflow.py b/modules/workflows/automation/mainWorkflow.py index 503d1d13..19cd1004 100644 --- a/modules/workflows/automation/mainWorkflow.py +++ b/modules/workflows/automation/mainWorkflow.py @@ -24,7 +24,7 @@ from .subAutomationUtils import parseScheduleToCron, planToPrompt, replacePlaceh logger = logging.getLogger(__name__) -async def chatStart(currentUser: User, userInput: UserInputRequest, workflowMode: WorkflowModeEnum, workflowId: Optional[str] = None) -> ChatWorkflow: +async def chatStart(currentUser: User, userInput: UserInputRequest, workflowMode: WorkflowModeEnum, workflowId: Optional[str] = None, mandateId: Optional[str] = None) -> ChatWorkflow: """ Starts a new chat or continues an existing one, then launches processing asynchronously. @@ -33,12 +33,13 @@ async def chatStart(currentUser: User, userInput: UserInputRequest, workflowMode userInput: User input request workflowId: Optional workflow ID to continue existing workflow workflowMode: "Dynamic" for iterative dynamic-style processing, "Automation" for automated workflow execution + mandateId: Mandate ID from request context (required for proper data isolation) Example usage for Dynamic mode: - workflow = await chatStart(currentUser, userInput, workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC) + workflow = await chatStart(currentUser, userInput, workflowMode=WorkflowModeEnum.WORKFLOW_DYNAMIC, mandateId=mandateId) """ try: - services = getServices(currentUser, None) + services = getServices(currentUser, mandateId=mandateId) workflowManager = WorkflowManager(services) workflow = await workflowManager.workflowStart(userInput, workflowMode, workflowId) return workflow @@ -46,10 +47,10 @@ async def chatStart(currentUser: User, userInput: UserInputRequest, workflowMode logger.error(f"Error starting chat: {str(e)}") raise -async def chatStop(currentUser: User, workflowId: str) -> ChatWorkflow: +async def chatStop(currentUser: User, workflowId: str, mandateId: Optional[str] = None) -> ChatWorkflow: """Stops a running chat.""" try: - services = getServices(currentUser, None) + services = getServices(currentUser, mandateId=mandateId) workflowManager = WorkflowManager(services) return await workflowManager.workflowStop(workflowId) except Exception as e: diff --git a/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py index e583d8bf..21de7537 100644 --- a/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py +++ b/modules/workflows/methods/methodSharepoint/actions/getExpensesFromPdf.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) # Configuration MAX_FILES_PER_EXECUTION = 50 +MAX_CONCURRENT_AI_TASKS = 10 # Limit concurrent AI calls to avoid rate limits ALLOWED_TAGS = ["customer", "meeting", "license", "subscription", "fuel", "food", "material"] RATE_LIMIT_WAIT_SECONDS = 60 @@ -92,6 +93,11 @@ async def getExpensesFromPdf(self, parameters: Dict[str, Any]) -> ActionResult: self.services.chat.progressLogFinish(operationId, False) return ActionResult.isFailure(error="No valid Microsoft connection found") + # Set access token for SharePoint service + if not self.services.sharepoint.setAccessTokenFromConnection(connection): + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="Failed to set SharePoint access token") + # Find site and folder info self.services.chat.progressLogUpdate(operationId, 0.1, "Resolving SharePoint site") siteInfo, folderPath = await _resolveSiteAndFolder(self, sharepointFolder) @@ -137,90 +143,104 @@ async def getExpensesFromPdf(self, parameters: Dict[str, Any]) -> ActionResult: featureInstanceId=featureInstanceId ) - # Process each PDF - for idx, pdfFile in enumerate(pdfFiles): - currentProgress = 0.2 + (idx * progressPerFile) + # Process PDFs in parallel with semaphore to limit concurrent AI calls + semaphore = asyncio.Semaphore(MAX_CONCURRENT_AI_TASKS) + completedCount = [0] # Use list for mutable reference in closure + + async def processSinglePdf(idx: int, pdfFile: Dict[str, Any]) -> Dict[str, Any]: + """Process a single PDF document. Returns result dict.""" fileName = pdfFile.get("name", f"file_{idx}") fileId = pdfFile.get("id") - self.services.chat.progressLogUpdate( - operationId, - currentProgress, - f"Processing {idx + 1}/{totalFiles}: {fileName}" - ) - - try: - # Download PDF content - fileContent = await self.services.sharepoint.downloadFile(siteId, fileId) - if not fileContent: - await _moveToErrorFolder(self, siteId, folderPath, fileName) - errorDocuments.append({ + async with semaphore: + # Update progress (thread-safe via asyncio) + completedCount[0] += 1 + currentProgress = 0.2 + (completedCount[0] * progressPerFile) + self.services.chat.progressLogUpdate( + operationId, + min(currentProgress, 0.9), + f"Processing {completedCount[0]}/{totalFiles}: {fileName}" + ) + + try: + # Download PDF content + fileContent = await self.services.sharepoint.downloadFile(siteId, fileId) + if not fileContent: + await _moveToErrorFolder(self, siteId, folderPath, fileName) + return {"type": "error", "file": fileName, "error": "Failed to download", "movedTo": "error/"} + + # AI call to extract expense data (this is the bottleneck - parallelized) + aiResult = await _extractExpensesWithAi(self.services, fileContent, fileName, prompt, featureInstanceId) + + if not aiResult.get("success"): + await _moveToErrorFolder(self, siteId, folderPath, fileName) + return {"type": "error", "file": fileName, "error": aiResult.get("error", "AI extraction failed"), "movedTo": "error/"} + + records = aiResult.get("records", []) + fileId = aiResult.get("fileId") + + # Check for empty records + if not records: + logger.warning(f"Document {fileName}: No records extracted, moving to error folder") + await _moveToErrorFolder(self, siteId, folderPath, fileName) + return {"type": "skipped", "file": fileName, "reason": "No expense records extracted", "movedTo": "error/"} + + # Validate and enrich records + validatedRecords = _validateAndEnrichRecords(records, fileName) + + # Save to TrusteePosition and create Document + Position-Document links + savedCount = _saveToTrusteePosition( + trusteeInterface, + validatedRecords, + featureInstanceId, + self.services.mandateId, + fileId=fileId, + fileName=fileName, + sourceLocation=sharepointFolder + ) + + # Move document to "processed" subfolder + timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + newFileName = f"{timestamp}_{fileName}" + + moveSuccess = await _moveToProcessedFolder(self, siteId, folderPath, fileName, newFileName) + + return { + "type": "processed", "file": fileName, - "error": "Failed to download", - "movedTo": "error/" - }) - continue - - # AI call to extract expense data - aiResult = await _extractExpensesWithAi(self.services, fileContent, fileName, prompt, featureInstanceId) - - if not aiResult.get("success"): + "newLocation": f"processed/{newFileName}" if moveSuccess else "move_failed", + "recordsExtracted": len(validatedRecords), + "recordsSaved": savedCount + } + + except Exception as e: + errorMsg = str(e) + logger.error(f"Error processing {fileName}: {errorMsg}") + + # Handle rate limit + if "429" in errorMsg or "throttl" in errorMsg.lower(): + logger.warning(f"Rate limit hit, waiting {RATE_LIMIT_WAIT_SECONDS} seconds") + await asyncio.sleep(RATE_LIMIT_WAIT_SECONDS) + await _moveToErrorFolder(self, siteId, folderPath, fileName) - errorDocuments.append({ - "file": fileName, - "error": aiResult.get("error", "AI extraction failed"), - "movedTo": "error/" - }) - continue - - records = aiResult.get("records", []) - - # Check for empty records - if not records: - logger.warning(f"Document {fileName}: No records extracted, moving to error folder") - await _moveToErrorFolder(self, siteId, folderPath, fileName) - skippedDocuments.append({ - "file": fileName, - "reason": "No expense records extracted", - "movedTo": "error/" - }) - continue - - # Validate and enrich records - validatedRecords = _validateAndEnrichRecords(records, fileName) - - # Save to TrusteePosition - savedCount = _saveToTrusteePosition(trusteeInterface, validatedRecords, featureInstanceId, self.services.mandateId) - totalPositions += savedCount - - # Move document to "processed" subfolder - timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") - newFileName = f"{timestamp}_{fileName}" - - moveSuccess = await _moveToProcessedFolder(self, siteId, folderPath, fileName, newFileName) - - processedDocuments.append({ - "file": fileName, - "newLocation": f"processed/{newFileName}" if moveSuccess else "move_failed", - "recordsExtracted": len(validatedRecords), - "recordsSaved": savedCount - }) - - except Exception as e: - errorMsg = str(e) - logger.error(f"Error processing {fileName}: {errorMsg}") - - # Handle rate limit - if "429" in errorMsg or "throttl" in errorMsg.lower(): - logger.warning(f"Rate limit hit, waiting {RATE_LIMIT_WAIT_SECONDS} seconds") - await asyncio.sleep(RATE_LIMIT_WAIT_SECONDS) - - await _moveToErrorFolder(self, siteId, folderPath, fileName) - errorDocuments.append({ - "file": fileName, - "error": errorMsg, - "movedTo": "error/" - }) + return {"type": "error", "file": fileName, "error": errorMsg, "movedTo": "error/"} + + # Execute all PDF processing tasks in parallel (limited by semaphore) + logger.info(f"Starting parallel processing of {totalFiles} PDFs (max {MAX_CONCURRENT_AI_TASKS} concurrent)") + tasks = [processSinglePdf(idx, pdfFile) for idx, pdfFile in enumerate(pdfFiles)] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Collect results + for result in results: + if isinstance(result, Exception): + errorDocuments.append({"file": "unknown", "error": str(result), "movedTo": "error/"}) + elif result.get("type") == "processed": + processedDocuments.append(result) + totalPositions += result.get("recordsSaved", 0) + elif result.get("type") == "skipped": + skippedDocuments.append(result) + elif result.get("type") == "error": + errorDocuments.append(result) # Create result summary self.services.chat.progressLogUpdate(operationId, 0.95, "Creating result summary") @@ -423,9 +443,10 @@ async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, pr # Step 5: Call AI with documentList - let AI service handle everything # (extraction, intent analysis, chunking, image processing) + # Use DATA_GENERATE (same path as ai.process) which handles chunking correctly options = AiCallOptions( resultFormat="csv", - operationType=OperationTypeEnum.DATA_EXTRACT + operationType=OperationTypeEnum.DATA_GENERATE ) aiResponse = await services.ai.callAiContent( @@ -433,17 +454,31 @@ async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, pr options=options, documentList=documentList, contentParts=None, # Let AI service extract from documents - outputFormat="csv" + outputFormat="csv", + generationIntent="extract" # Signal this is extraction, not document generation ) - if not aiResponse or not aiResponse.content: + if not aiResponse: return {"success": False, "error": "AI returned empty response"} - # Parse CSV response - csvContent = aiResponse.content + # Get CSV from rendered documents (not from content - that's the internal structure) + if not aiResponse.documents or len(aiResponse.documents) == 0: + return {"success": False, "error": "AI returned no documents"} + + # Get the CSV content from the first document + csvDocument = aiResponse.documents[0] + csvContent = csvDocument.documentData + + # documentData is bytes, decode to string + if isinstance(csvContent, bytes): + csvContent = csvContent.decode('utf-8') + + logger.info(f"Retrieved CSV content ({len(csvContent)} chars) from rendered document: {csvDocument.documentName}") + records = _parseCsvToRecords(csvContent) - return {"success": True, "records": records} + # Return fileId so it can be used to create TrusteeDocument reference + return {"success": True, "records": records, "fileId": fileItem.id} except Exception as e: logger.error(f"AI extraction error for {fileName}: {str(e)}") @@ -454,8 +489,9 @@ def _parseCsvToRecords(csvContent: str) -> List[Dict[str, Any]]: """Parse CSV content to list of expense records.""" records = [] try: - # Clean up CSV content - remove markdown code blocks if present content = csvContent.strip() + + # Clean up CSV content - remove markdown code blocks if present if content.startswith("```"): lines = content.split('\n') # Remove first and last line if they're code block markers @@ -470,6 +506,8 @@ def _parseCsvToRecords(csvContent: str) -> List[Dict[str, Any]]: # Clean up keys (remove whitespace) cleanedRow = {k.strip(): v.strip() if isinstance(v, str) else v for k, v in row.items()} records.append(cleanedRow) + + logger.info(f"Parsed {len(records)} records from CSV content") except Exception as e: logger.error(f"Error parsing CSV: {str(e)}") @@ -548,10 +586,54 @@ def _parseFloat(value) -> float: return 0.0 -def _saveToTrusteePosition(trusteeInterface, records: List[Dict[str, Any]], featureInstanceId: str, mandateId: str) -> int: - """Save validated records to TrusteePosition table.""" - savedCount = 0 +def _saveToTrusteePosition( + trusteeInterface, + records: List[Dict[str, Any]], + featureInstanceId: str, + mandateId: str, + fileId: Optional[str] = None, + fileName: Optional[str] = None, + sourceLocation: Optional[str] = None +) -> int: + """ + Save validated records to TrusteePosition table. + Also creates TrusteeDocument (referencing the source file) and links positions to it. + Args: + trusteeInterface: Trustee interface instance + records: List of expense records to save + featureInstanceId: Feature instance ID + mandateId: Mandate ID + fileId: Optional file ID from central Files table (source PDF) + fileName: Optional file name + sourceLocation: Optional source location (e.g., SharePoint path) + + Returns: + Number of positions saved + """ + savedCount = 0 + savedPositionIds = [] + + # Step 1: Create TrusteeDocument referencing the source file + documentId = None + if fileId and fileName: + try: + document = trusteeInterface.createDocument({ + "fileId": fileId, + "documentName": fileName, + "documentMimeType": "application/pdf", + "sourceType": "sharepoint", + "sourceLocation": sourceLocation + }) + if document: + documentId = document.id + logger.info(f"Created TrusteeDocument {documentId} referencing file {fileId}") + else: + logger.warning(f"Failed to create TrusteeDocument for file {fileId}") + except Exception as e: + logger.error(f"Error creating TrusteeDocument: {str(e)}") + + # Step 2: Save positions for record in records: try: position = { @@ -573,11 +655,27 @@ def _saveToTrusteePosition(trusteeInterface, records: List[Dict[str, Any]], feat result = trusteeInterface.createPosition(position) if result: savedCount += 1 + savedPositionIds.append(result.id) logger.debug(f"Saved position: {position.get('company')} - {position.get('bookingAmount')}") except Exception as e: logger.error(f"Failed to save position: {str(e)}") + # Step 3: Create Position-Document links + if documentId and savedPositionIds: + for positionId in savedPositionIds: + try: + link = trusteeInterface.createPositionDocument({ + "documentId": documentId, + "positionId": positionId + }) + if link: + logger.debug(f"Created position-document link: {positionId} -> {documentId}") + else: + logger.warning(f"Failed to create position-document link: {positionId} -> {documentId}") + except Exception as e: + logger.error(f"Error creating position-document link: {str(e)}") + return savedCount @@ -718,27 +816,16 @@ async def _deleteFile(self, siteId: str, folderPath: str, fileName: str) -> bool if not fileId: return False - # Delete by ID + # Delete by ID using apiClient deleteEndpoint = f"sites/{siteId}/drive/items/{fileId}" + result = await self.apiClient.makeGraphApiCall(deleteEndpoint, method="DELETE") - # Make DELETE request - if self.services.sharepoint.accessToken is None: - logger.error("Access token not set for delete") + if "error" in result: + logger.warning(f"Delete failed: {result['error']}") return False - import aiohttp - headers = {"Authorization": f"Bearer {self.services.sharepoint.accessToken}"} - url = f"https://graph.microsoft.com/v1.0/{deleteEndpoint}" - - async with aiohttp.ClientSession() as session: - async with session.delete(url, headers=headers) as response: - if response.status in [200, 204]: - logger.debug(f"Deleted file: {filePath}") - return True - else: - errorText = await response.text() - logger.warning(f"Delete failed: {response.status} - {errorText}") - return False + logger.debug(f"Deleted file: {filePath}") + return True except Exception as e: logger.error(f"Failed to delete file: {str(e)}") diff --git a/modules/workflows/methods/methodSharepoint/helpers/apiClient.py b/modules/workflows/methods/methodSharepoint/helpers/apiClient.py index 7cead7ef..542e6dde 100644 --- a/modules/workflows/methods/methodSharepoint/helpers/apiClient.py +++ b/modules/workflows/methods/methodSharepoint/helpers/apiClient.py @@ -92,6 +92,18 @@ class ApiClientHelper: errorText = await response.text() logger.error(f"Graph API call failed: {response.status} - {errorText}") return {"error": f"API call failed: {response.status} - {errorText}"} + + elif method == "DELETE": + logger.debug(f"Starting DELETE request to {url}") + async with session.delete(url, headers=headers) as response: + logger.info(f"Graph API response: {response.status}") + if response.status in [200, 204]: + logger.debug(f"Graph API DELETE success") + return {"success": True} + else: + errorText = await response.text() + logger.error(f"Graph API call failed: {response.status} - {errorText}") + return {"error": f"API call failed: {response.status} - {errorText}"} except asyncio.TimeoutError: logger.error(f"Graph API call timed out after 30 seconds: {endpoint}") From 7ca957f664d7c0d4574a9c014b2e881b5b71cc1b Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 23:40:12 +0100 Subject: [PATCH 30/32] fixed filter/sort for rtustee --- .../trustee/interfaceFeatureTrustee.py | 223 +++++++++++++++++- 1 file changed, 217 insertions(+), 6 deletions(-) diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index c3400752..99553108 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -212,6 +212,197 @@ class TrusteeObjects: return getattr(permissions, operation, AccessLevel.NONE) + # ===== Pagination Helper Functions ===== + + def _applyFilters(self, records: List[Dict[str, Any]], params: Optional[PaginationParams]) -> List[Dict[str, Any]]: + """ + Apply filter criteria to records. + + Supports: + - General search: params.filters["search"] - searches across all text fields + - Field-specific filters: params.filters + - Simple: {"status": "running"} - equals match + - With operator: {"status": {"operator": "equals", "value": "running"}} + - Operators: equals, contains, gt, gte, lt, lte, in, notIn, startsWith, endsWith + + Args: + records: List of record dictionaries to filter + params: PaginationParams with filters (search is inside filters) + + Returns: + Filtered list of records + """ + if not params or not records: + return records + + # Get filters safely (may be None) + filters = getattr(params, 'filters', None) + if not filters: + return records + + filtered = records + + # Handle general search across text fields (search is inside filters) + searchTerm = filters.get("search") if isinstance(filters, dict) else None + if searchTerm: + searchTerm = str(searchTerm).lower() + if searchTerm: + searchFiltered = [] + for record in filtered: + found = False + for key, value in record.items(): + if isinstance(value, str) and searchTerm in value.lower(): + found = True + break + elif isinstance(value, (int, float)) and searchTerm in str(value): + found = True + break + if found: + searchFiltered.append(record) + filtered = searchFiltered + + # Handle field-specific filters + if filters: + for fieldName, filterValue in filters.items(): + if fieldName == "search": + continue # Already handled above + + fieldFiltered = [] + for record in filtered: + if fieldName not in record: + continue + + recordValue = record.get(fieldName) + + # Handle simple value (equals operator) + if not isinstance(filterValue, dict): + if recordValue == filterValue: + fieldFiltered.append(record) + continue + + # Handle filter with operator + operator = filterValue.get("operator", "equals") + filterVal = filterValue.get("value") + + matches = False + if operator in ["equals", "eq"]: + matches = recordValue == filterVal + + elif operator == "contains": + recordStr = str(recordValue).lower() if recordValue is not None else "" + filterStr = str(filterVal).lower() if filterVal is not None else "" + matches = filterStr in recordStr + + elif operator == "startsWith": + recordStr = str(recordValue).lower() if recordValue is not None else "" + filterStr = str(filterVal).lower() if filterVal is not None else "" + matches = recordStr.startswith(filterStr) + + elif operator == "endsWith": + recordStr = str(recordValue).lower() if recordValue is not None else "" + filterStr = str(filterVal).lower() if filterVal is not None else "" + matches = recordStr.endswith(filterStr) + + elif operator == "gt": + try: + recordNum = float(recordValue) if recordValue is not None else float('-inf') + filterNum = float(filterVal) if filterVal is not None else float('-inf') + matches = recordNum > filterNum + except (ValueError, TypeError): + matches = False + + elif operator == "gte": + try: + recordNum = float(recordValue) if recordValue is not None else float('-inf') + filterNum = float(filterVal) if filterVal is not None else float('-inf') + matches = recordNum >= filterNum + except (ValueError, TypeError): + matches = False + + elif operator == "lt": + try: + recordNum = float(recordValue) if recordValue is not None else float('inf') + filterNum = float(filterVal) if filterVal is not None else float('inf') + matches = recordNum < filterNum + except (ValueError, TypeError): + matches = False + + elif operator == "lte": + try: + recordNum = float(recordValue) if recordValue is not None else float('inf') + filterNum = float(filterVal) if filterVal is not None else float('inf') + matches = recordNum <= filterNum + except (ValueError, TypeError): + matches = False + + elif operator == "in": + if isinstance(filterVal, list): + matches = recordValue in filterVal + else: + matches = False + + elif operator == "notIn": + if isinstance(filterVal, list): + matches = recordValue not in filterVal + else: + matches = False + + if matches: + fieldFiltered.append(record) + + filtered = fieldFiltered + + return filtered + + def _applySorting(self, records: List[Dict[str, Any]], params: Optional[PaginationParams]) -> List[Dict[str, Any]]: + """Apply multi-level sorting to records using stable sort.""" + if not params: + return records + + # Get sort safely (may be None or empty list) + sortFields = getattr(params, 'sort', None) + if not sortFields: + return records + + sortedRecords = list(records) + + # Sort from least significant to most significant field (reverse order) + # Python's sort is stable, so this creates proper multi-level sorting + for sortField in reversed(sortFields): + # Handle both dict and object formats + if isinstance(sortField, dict): + fieldName = sortField.get("field") + direction = sortField.get("direction", "asc") + else: + fieldName = getattr(sortField, "field", None) + direction = getattr(sortField, "direction", "asc") + + if not fieldName: + continue + + isDesc = (direction == "desc") + + def makeSortKey(fName): + def sortKey(record): + value = record.get(fName) + # Handle None values - place them at the end for both directions + if value is None: + return (1, "") # sorts after (0, ...) + else: + if isinstance(value, (int, float)): + return (0, value) + elif isinstance(value, str): + return (0, value.lower()) + elif isinstance(value, bool): + return (0, value) + else: + return (0, str(value)) + return sortKey + + sortedRecords.sort(key=makeSortKey(fieldName), reverse=isDesc) + + return sortedRecords + # ===== Organisation CRUD ===== def createOrganisation(self, data: Dict[str, Any]) -> Optional[TrusteeOrganisation]: @@ -819,12 +1010,22 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Convert dicts to Pydantic objects (remove binary data and internal fields) - pydanticItems = [] + # Clean records (remove binary data and internal fields) - keep as dicts for filtering/sorting + cleanedRecords = [] for record in records: cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} - pydanticItems.append(TrusteeDocument(**cleanedRecord)) + cleanedRecords.append(cleanedRecord) + # Step 2: Apply filters (search and field filters) + filteredRecords = self._applyFilters(cleanedRecords, params) + + # Step 3: Apply sorting + sortedRecords = self._applySorting(filteredRecords, params) + + # Step 4: Convert to Pydantic objects + pydanticItems = [TrusteeDocument(**r) for r in sortedRecords] + + # Step 5: Apply pagination totalItems = len(pydanticItems) if params: pageSize = params.pageSize or 20 @@ -965,12 +1166,22 @@ class TrusteeObjects: featureInstanceId=self.featureInstanceId ) - # Convert dicts to Pydantic objects (remove internal fields) - pydanticItems = [] + # Clean records (remove internal fields) - keep as dicts for filtering/sorting + cleanedRecords = [] for record in records: cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} - pydanticItems.append(TrusteePosition(**cleanedRecord)) + cleanedRecords.append(cleanedRecord) + # Step 2: Apply filters (search and field filters) + filteredRecords = self._applyFilters(cleanedRecords, params) + + # Step 3: Apply sorting + sortedRecords = self._applySorting(filteredRecords, params) + + # Step 4: Convert to Pydantic objects + pydanticItems = [TrusteePosition(**r) for r in sortedRecords] + + # Step 5: Apply pagination totalItems = len(pydanticItems) if params: pageSize = params.pageSize or 20 From ee15fd64b07bb9b21228d1f6152355cf701740ea Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 26 Jan 2026 23:48:19 +0100 Subject: [PATCH 31/32] fix crosstable trustee --- .../trustee/interfaceFeatureTrustee.py | 24 +++++++++++++++++-- modules/routes/routeAdminFeatures.py | 24 ++++++++++++++----- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index 99553108..b5e08e2b 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -1095,7 +1095,8 @@ class TrusteeObjects: def deleteDocument(self, documentId: str) -> bool: """Delete a document. - Note: organisationId and contractId removed - feature instance IS the organisation. + All position-document cross-table entries (TrusteePositionDocument) referencing + this document are deleted first, then the document. """ # Get existing document to check creator existingRecords = self.db.getRecordset(TrusteeDocument, recordFilter={"id": documentId}) @@ -1112,6 +1113,7 @@ class TrusteeObjects: logger.warning(f"User {self.userId} lacks permission to delete document") return False + self._deletePositionDocumentLinksForDocument(documentId) return self.db.recordDelete(TrusteeDocument, documentId) # ===== Position CRUD ===== @@ -1259,7 +1261,8 @@ class TrusteeObjects: def deletePosition(self, positionId: str) -> bool: """Delete a position. - Note: organisationId and contractId removed - feature instance IS the organisation. + All position-document cross-table entries (TrusteePositionDocument) referencing + this position are deleted first, then the position. """ # Get existing position to check creator existingRecords = self.db.getRecordset(TrusteePosition, recordFilter={"id": positionId}) @@ -1276,6 +1279,7 @@ class TrusteeObjects: logger.warning(f"User {self.userId} lacks permission to delete position") return False + self._deletePositionDocumentLinksForPosition(positionId) return self.db.recordDelete(TrusteePosition, positionId) # ===== Position-Document Link CRUD ===== @@ -1423,6 +1427,22 @@ class TrusteeObjects: return self.db.recordDelete(TrusteePositionDocument, linkId) + def _deletePositionDocumentLinksForDocument(self, documentId: str) -> None: + """Delete all position-document cross-table entries referencing this document.""" + links = self.db.getRecordset(TrusteePositionDocument, recordFilter={"documentId": documentId}) + for link in links: + linkId = link.get("id") + if linkId: + self.db.recordDelete(TrusteePositionDocument, linkId) + + def _deletePositionDocumentLinksForPosition(self, positionId: str) -> None: + """Delete all position-document cross-table entries referencing this position.""" + links = self.db.getRecordset(TrusteePositionDocument, recordFilter={"positionId": positionId}) + for link in links: + linkId = link.get("id") + if linkId: + self.db.recordDelete(TrusteePositionDocument, linkId) + # ===== Trustee-specific Access Check ===== def getUserAccessForOrganisation(self, userId: str, organisationId: str) -> List[Dict[str, Any]]: diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 82b796c1..56a79741 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -878,6 +878,12 @@ class FeatureInstanceUserResponse(BaseModel): enabled: bool +class FeatureInstanceUserUpdate(BaseModel): + """Request model for updating a feature instance user (roles and active flag)""" + roleIds: List[str] = Field(..., description="Role IDs to assign") + enabled: Optional[bool] = Field(None, description="Whether this user's access is active (omit to leave unchanged)") + + @router.get("/instances/{instanceId}/users", response_model=List[FeatureInstanceUserResponse]) @limiter.limit("60/minute") async def list_feature_instance_users( @@ -1161,18 +1167,19 @@ async def update_feature_instance_user_roles( request: Request, instanceId: str, userId: str, - roleIds: List[str], + data: FeatureInstanceUserUpdate, context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """ - Update a user's roles in a feature instance. + Update a user's roles and active flag in a feature instance. Replaces all existing FeatureAccessRole records with new ones. + If enabled is provided, updates the FeatureAccess.enabled flag. Args: instanceId: FeatureInstance ID userId: User ID to update - roleIds: New list of role IDs + data: roleIds and optional enabled """ try: rootInterface = getRootInterface() @@ -1215,6 +1222,10 @@ async def update_feature_instance_user_roles( featureAccessId = existingAccess[0].get("id") + # Update enabled flag if provided + if data.enabled is not None: + rootInterface.db.recordModify(FeatureAccess, featureAccessId, {"enabled": data.enabled}) + # Delete existing FeatureAccessRole records existingRoles = rootInterface.db.getRecordset( FeatureAccessRole, @@ -1224,7 +1235,7 @@ async def update_feature_instance_user_roles( rootInterface.db.recordDelete(FeatureAccessRole, role.get("id")) # Create new FeatureAccessRole records - for roleId in roleIds: + for roleId in data.roleIds: featureAccessRole = FeatureAccessRole( featureAccessId=featureAccessId, roleId=roleId @@ -1232,14 +1243,15 @@ async def update_feature_instance_user_roles( rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) logger.info( - f"User {context.user.id} updated roles for user {userId} in feature instance {instanceId}: {roleIds}" + f"User {context.user.id} updated roles for user {userId} in feature instance {instanceId}: {data.roleIds}" ) return { "featureAccessId": featureAccessId, "userId": userId, "featureInstanceId": instanceId, - "roleIds": roleIds + "roleIds": data.roleIds, + "enabled": data.enabled if data.enabled is not None else existingAccess[0].get("enabled", True) } except HTTPException: From 4c91bd76077f786495066b7f12db87a17fbfaf6b Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 27 Jan 2026 00:28:31 +0100 Subject: [PATCH 32/32] fixes --- modules/interfaces/interfaceDbApp.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 250b2a38..1f1d1e53 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -742,8 +742,16 @@ class AppObjects: logger.error(f"Unexpected error creating user: {str(e)}") raise ValueError(f"Failed to create user: {str(e)}") - def updateUser(self, userId: str, updateData: Union[Dict[str, Any], User]) -> User: - """Update a user's information""" + def updateUser(self, userId: str, updateData: Union[Dict[str, Any], User], allowSysAdminChange: bool = False) -> User: + """Update a user's information. + + Args: + userId: ID of the user to update + updateData: User data to update (dict or User model) + allowSysAdminChange: If True, allows changing isSysAdmin field. + Only set to True when called by a SysAdmin explicitly + changing another user's admin status. + """ try: # Get user user = self.getUser(userId) @@ -758,6 +766,14 @@ class AppObjects: # Remove id field from updateDict if present - we'll use userId from parameter updateDict.pop("id", None) + + # SECURITY: Protect sensitive fields from being overwritten by profile updates. + # These fields should only be changed explicitly by admins, not through + # profile forms where they might be sent as default values (e.g., isSysAdmin=False). + protectedFields = ["isSysAdmin"] + if not allowSysAdminChange: + for field in protectedFields: + updateDict.pop(field, None) # Update user data using model updatedData = user.model_dump()