From ccc41e70237767aad293a7ea16fa57d1e2144390 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 19 Jan 2026 09:18:37 +0100 Subject: [PATCH] harmonized module names --- app.py | 6 +- .../migration_export_20260119_085558.json | 1221 +++++++++++++++++ modules/datamodels/datamodelAudit.py | 208 +++ .../{datamodelChat.py => datamodelChatbot.py} | 36 + modules/datamodels/datamodelFiles.py | 2 + modules/datamodels/datamodelMessaging.py | 25 + modules/datamodels/datamodelNeutralizer.py | 4 + modules/datamodels/datamodelRealEstate.py | 40 + modules/datamodels/datamodelTrustee.py | 70 + modules/datamodels/datamodelVoice.py | 2 + modules/datamodels/datamodelWorkflow.py | 2 +- .../datamodels/datamodelWorkflowActions.py | 2 +- modules/features/chatbot/mainChatbot.py | 4 +- modules/features/realEstate/mainRealEstate.py | 2 +- modules/features/workflow/mainWorkflow.py | 2 +- .../features/workflow/subAutomationUtils.py | 2 +- ...DbChatObjects.py => interfaceDbChatbot.py} | 70 +- .../interfaces/interfaceDbComponentObjects.py | 35 +- ...ateObjects.py => interfaceDbRealEstate.py} | 40 +- ...rusteeObjects.py => interfaceDbTrustee.py} | 37 +- modules/routes/routeDataAutomation.py | 4 +- modules/routes/routeDataMandates.py | 28 +- modules/routes/routeDataUsers.py | 8 +- modules/routes/routeFeatureChatDynamic.py | 6 +- modules/routes/routeFeatureChatbot.py | 6 +- modules/routes/routeFeatureRealEstate.py | 2 +- modules/routes/routeFeatureTrustee.py | 364 +++-- ...eAutomation.py => routeFeatureWorkflow.py} | 6 +- modules/routes/routeGdpr.py | 21 +- modules/routes/routeSecurityGoogle.py | 5 +- modules/routes/routeSecurityLocal.py | 21 +- modules/routes/routeSecurityMsft.py | 5 +- modules/routes/routeWorkflows.py | 8 +- modules/services/__init__.py | 6 +- modules/services/serviceAi/mainServiceAi.py | 2 +- .../serviceAi/subContentExtraction.py | 2 +- .../services/serviceAi/subDocumentIntents.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../mainServiceExtraction.py | 2 +- .../mainServiceGeneration.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 6 +- modules/shared/auditLogger.py | 548 ++++++-- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 8 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 12 +- modules/workflows/workflowManager.py | 10 +- tests/functional/test02_ai_models.py | 2 +- tests/functional/test03_ai_operations.py | 18 +- tests/functional/test04_ai_behavior.py | 6 +- .../test05_workflow_with_documents.py | 8 +- .../test06_workflow_prompt_variations.py | 8 +- .../test09_document_generation_formats.py | 8 +- .../test10_document_generation_formats.py | 8 +- .../test11_code_generation_formats.py | 8 +- .../workflows/test_workflow_execution.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- tool_db_export_migration.py | 508 +++++++ tool_db_import_migration.py | 612 +++++++++ 100 files changed, 3756 insertions(+), 432 deletions(-) create mode 100644 local/backup/migration_export_20260119_085558.json create mode 100644 modules/datamodels/datamodelAudit.py rename modules/datamodels/{datamodelChat.py => datamodelChatbot.py} (95%) rename modules/interfaces/{interfaceDbChatObjects.py => interfaceDbChatbot.py} (95%) rename modules/interfaces/{interfaceDbRealEstateObjects.py => interfaceDbRealEstate.py} (93%) rename modules/interfaces/{interfaceDbTrusteeObjects.py => interfaceDbTrustee.py} (96%) rename modules/routes/{routeFeatureAutomation.py => routeFeatureWorkflow.py} (95%) create mode 100644 tool_db_export_migration.py create mode 100644 tool_db_import_migration.py diff --git a/app.py b/app.py index 7ed57ed9..b489194d 100644 --- a/app.py +++ b/app.py @@ -292,6 +292,10 @@ async def lifespan(app: FastAPI): # --- Init Managers --- await featuresLifecycle.start(eventUser) eventManager.start() + + # Register audit log cleanup scheduler + from modules.shared.auditLogger import registerAuditLogCleanupScheduler + registerAuditLogCleanupScheduler() yield @@ -444,7 +448,7 @@ app.include_router(sharepointRouter) from modules.routes.routeDataAutomation import router as automationRouter app.include_router(automationRouter) -from modules.routes.routeFeatureAutomation import router as adminAutomationEventsRouter +from modules.routes.routeFeatureWorkflow import router as adminAutomationEventsRouter app.include_router(adminAutomationEventsRouter) from modules.routes.routeRbac import router as rbacRouter diff --git a/local/backup/migration_export_20260119_085558.json b/local/backup/migration_export_20260119_085558.json new file mode 100644 index 00000000..50cc9e5a --- /dev/null +++ b/local/backup/migration_export_20260119_085558.json @@ -0,0 +1,1221 @@ +{ + "meta": { + "exportedAt": "2026-01-19T07:55:59.185004Z", + "exportedFrom": "Development Instance Patrick", + "databaseName": "poweron_app", + "version": "1.0", + "tableCount": 6, + "excludedTables": [ + "_system" + ], + "includesMeta": false, + "totalRecords": 107 + }, + "tables": { + "AccessRule": [ + { + "id": "90990e75-ac45-4986-9c09-f299f198d2b3", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": null, + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "a9536849-a53b-458d-bab8-77a2a3ac2747", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": null, + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "n" + }, + { + "id": "54870935-d5a1-41b7-b088-30f70b4dc835", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": null, + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "a3743435-1015-4bd1-a256-4f91eda9f544", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": null, + "view": 1, + "read": "g", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "3e7b79b5-a68a-41b5-9e7f-f8159a310ed3", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Mandate", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "e7e8818c-04ed-473a-b1da-c7095f98f99e", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "61714126-2822-48cd-bbb5-175c141b2c0b", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0ae9dcb8-302a-44cb-b777-1f9d26af7190", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Mandate", + "view": 0, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "2ab27800-df9d-4307-aee3-d99064fbab5d", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "279c6538-401a-4cd3-a36a-d1399f8bd2e4", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "4ab1abbe-abf1-4510-8ba1-4afbb37b1fd1", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "m", + "create": "n", + "update": "m", + "delete": "n" + }, + { + "id": "86c384a7-a2b5-4207-9e62-42cc50a2d20b", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "UserInDB", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "542af362-50c8-4e19-af21-5db2e0b8733e", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "8e356f1c-134d-4115-962e-0d0b7b71cf65", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "061b7a09-1003-4250-9524-64d648135467", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "89704bf9-d742-4149-a1e2-dcaaba9957c9", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "UserConnection", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "d296b5fb-48c3-4351-aa84-70ec1593369e", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "6ac892f8-69b6-4c2f-b18c-f5d58f8c95d9", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "defc2070-098f-44e2-9c59-264189730cc7", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "5927b71b-51f7-4671-a91b-f4f300af04f7", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "DataNeutraliserConfig", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "5a1907e7-6772-4b3f-9d34-6b847db3c14c", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "c4eac0f8-e9d2-4064-80bd-a4e9c9e68686", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "77a76ccf-91bb-4348-9caf-912b2bd7fe02", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "d059cf52-8c56-4273-80f9-e7b8b40ebb4b", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "DataNeutralizerAttributes", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0ca0ed8b-9327-43ff-b7b5-dc5870fb09c2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "a72168c5-bf33-43ef-a708-fd40e8feb6ba", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "9c80b57b-d2ed-4309-9e87-62a98fe144d0", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "4e534940-33cc-43ac-b859-9360a2e60cf1", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "ChatWorkflow", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "b0b3e37e-ea48-471e-95f6-b8e1e62ad09b", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "0f0f4c89-2aa5-4755-aeea-562bd020593d", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "0185dcc1-3560-4255-9840-8ab5db8042f8", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "c9c87948-21ce-47f4-8040-c02a0917aeb2", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Prompt", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "9e5f6dd5-c7a7-4a22-a1d6-82d4c68630a0", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "852f4f77-dac7-46de-8136-ebc46344101e", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "61710630-3c71-4adf-a997-b9e61d06fc00", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "158789bd-7f26-4d88-a27e-6ce11b3d94f9", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Projekt", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "c96b8b90-df7b-49ea-8ec9-0754c6179561", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "cdb610f9-21c6-4e8d-b9b6-d4a44f372ad6", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "6d3ff830-ae8f-4447-89a8-eaa8f8c4ecc7", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "7791781c-810a-43ac-a5ce-d84a3584e5f4", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Parzelle", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "f49cd1cb-bdcd-4a9a-92fb-205078a3acba", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "44d5455e-a17b-4b6d-a426-21565fe1cef7", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "b2b58648-a555-4135-a038-84218a279437", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "073195c6-01bc-40f9-9896-f3c10ebce288", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Dokument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "ba537ef0-239b-456d-b354-dec2c9d921ed", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "3d66cd04-f4e6-43f8-ab77-c31a87730096", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "bcc52c15-b41c-4a97-8e13-10035dd22202", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "844c7c02-dcc8-43a4-8eb5-f8ec198f50fd", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Gemeinde", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "15c1c76d-b8f7-40d2-8027-5ffdf6c96e28", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "b73379ef-3dbf-4118-b6f2-473f3c1c52e7", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "afc14c03-e953-4186-a923-4a12d3c6a312", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "c49ee8f3-f295-465d-be92-9bf46ee259e0", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Kanton", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "fcadb856-46f4-4ff0-85eb-67df3891c25d", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "4a4e52ad-4ee3-4044-9e14-e0d797356139", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "297ae474-ee2d-42f0-af09-eaa8b48d9684", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "a567d373-00be-4c7e-b25e-deb6b10b57a3", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "Land", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "8f961779-1790-4cad-bebc-9b9bc436a29c", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "81be065a-9ec6-4713-9fca-f5087cb9c3ee", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "1fe2a920-0f9d-4c33-9288-4381af8c523e", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "e99b39eb-ed55-4d08-a5d5-c3cf4645c312", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeOrganisation", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "a54eed2a-e269-4884-ac2f-c09990f9b3d2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "2bd073ca-30c3-43e0-8ea1-d77241053e7c", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "557ba1d5-2be0-43ee-a842-d0e74b45df41", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "81b46bcc-6cc3-40e0-9d82-f3a5bcc23b53", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeRole", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "f3a58145-8986-42a7-bce4-eab0a1fa0962", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "31d2a772-ed6b-447b-aa95-16a160d39601", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "8a373342-8b8b-4a64-afd4-9f3ae8071d28", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "b7a501e1-793f-47b6-b20c-1bfa61531cee", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeAccess", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "0e67e101-5ce6-40fb-a783-104eebe29f92", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "23005b91-dc46-4bbe-a690-6c4568484858", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "aec6753b-cd7d-4ec8-8fa6-b9d007eb3657", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "31bf3c95-1012-4d61-91fc-36e4e3b832ec", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeContract", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "17edb067-e62e-489a-842b-adbe4588aec0", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "981e57b7-d813-4d67-9cf9-73576cc2affb", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "3078980e-e8f2-44f0-b172-4a4c877b1b14", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "f25efabc-e861-4cba-ad9c-6c8886f99ae5", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteeDocument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "178c1dc4-7879-4a98-bbdd-54c1c55500c2", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "bbf02493-5ba2-4c0d-83b7-2a1c20e41108", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "7183b536-9a8a-4c2e-b6f4-4f69cc8a50b4", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "309130ca-254d-45b0-8629-47fe6a391a34", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteePosition", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "240fdaa4-75df-47a4-b33e-bce2e09dc741", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "a", + "create": "a", + "update": "a", + "delete": "a" + }, + { + "id": "528ba11a-824d-477a-ae2f-aa453fdea2f2", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "g", + "create": "g", + "update": "g", + "delete": "g" + }, + { + "id": "e3efeb99-eeb2-4450-85a4-f4261449a9ea", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "m", + "create": "m", + "update": "m", + "delete": "m" + }, + { + "id": "ca4bf93c-311c-4d2b-8b6e-9bd879ff5cde", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "TrusteePositionDocument", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "186bc21d-eb17-4d4a-ae4e-20725768befb", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "a", + "create": "n", + "update": "n", + "delete": "a" + }, + { + "id": "73d6eaa4-c1d1-4bfb-a2d6-a1772f9fd31c", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "a", + "create": "n", + "update": "n", + "delete": "a" + }, + { + "id": "2dce96a0-9a30-4d83-8544-29b60b7e8199", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "91b2e3ef-7472-41b0-81ef-7e891ce7d183", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "DATA", + "item": "AuthEvent", + "view": 1, + "read": "m", + "create": "n", + "update": "n", + "delete": "n" + }, + { + "id": "5bf4d7af-d8e9-4ca0-a9d1-c4cdd6a8b2a8", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "37999b49-c13c-44a1-a94e-41477d2e2e29", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "b7af378c-0da9-4554-ab9e-3e4d5130778b", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "d0c5bc55-d6e7-40c2-8bab-b7df13836712", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "UI", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "96827d13-bb10-4341-9615-03b940e4eab1", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "c052f46c-07b7-447b-885d-15803f615b6a", + "roleId": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "91213ddd-9490-4f9f-928a-5f7c87bb6e05", + "roleId": "bc22885c-5354-463e-a3fe-480941e016df", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + }, + { + "id": "58abcdc0-c8ba-435c-b01e-7b9c52581251", + "roleId": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "context": "RESOURCE", + "item": null, + "view": 1, + "read": null, + "create": null, + "update": null, + "delete": null + } + ], + "Mandate": [ + { + "id": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "name": "Root", + "enabled": 1 + } + ], + "Role": [ + { + "id": "4564beaf-f07d-420f-a8af-d46dff0ba3b3", + "roleLabel": "sysadmin", + "description": { + "en": "System Administrator - Full access to all system resources", + "fr": "Administrateur système - Accès complet à toutes les ressources", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "9d7af325-2dc9-451f-88d1-090dc06de3db", + "roleLabel": "admin", + "description": { + "en": "Administrator - Manage users and resources within mandate scope", + "fr": "Administrateur - Gérer les utilisateurs et ressources dans le périmètre du mandat", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "bc22885c-5354-463e-a3fe-480941e016df", + "roleLabel": "user", + "description": { + "en": "User - Standard user with access to own records", + "fr": "Utilisateur - Utilisateur standard avec accès à ses propres enregistrements", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + }, + { + "id": "95a88cf7-8a2a-42b2-b136-168966ad86b5", + "roleLabel": "viewer", + "description": { + "en": "Viewer - Read-only access to group records", + "fr": "Visualiseur - Accès en lecture seule aux enregistrements du groupe", + "ge": null, + "it": null + }, + "mandateId": null, + "featureInstanceId": null, + "featureCode": null, + "isSystemRole": 1 + } + ], + "UserInDB": [ + { + "id": "3876d530-29d8-451d-a2fc-92af5cb2b817", + "username": "admin", + "email": "admin@example.com", + "fullName": "Administrator", + "language": "en", + "enabled": 1, + "isSysAdmin": 1, + "roleLabels": [ + "sysadmin" + ], + "authenticationAuthority": "local", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$Sumds1bqvZfyPseYs/YeYw$eiRcnO7J+ebit2oV6Ndqaer2ZIgPErTC9q2riRpiiwA", + "resetToken": null, + "resetTokenExpires": null + }, + { + "id": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", + "username": "event", + "email": "event@example.com", + "fullName": "Event", + "language": "en", + "enabled": 1, + "isSysAdmin": 1, + "roleLabels": [ + "sysadmin" + ], + "authenticationAuthority": "local", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "hashedPassword": "$argon2id$v=19$m=65536,t=3,p=4$BoDwnlNq7d3b2ztnrPWecw$ICkAaTjE/R39CJ7MryLmfmeEX5m4N/6S3HaDfOZuOBM", + "resetToken": null, + "resetTokenExpires": null + } + ], + "UserMandate": [ + { + "id": "70f9e733-7297-4afe-8784-9f7c730de3c4", + "userId": "3876d530-29d8-451d-a2fc-92af5cb2b817", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "enabled": 1 + }, + { + "id": "412ba93d-2abe-4916-8a80-bec4672c0baf", + "userId": "805dcd6a-ac87-48c4-a1f0-987d8aa4a64b", + "mandateId": "e439ce2b-a8c2-4684-8c5f-70df493b82a1", + "enabled": 1 + } + ], + "UserMandateRole": [ + { + "id": "cff2dd70-4dd6-4028-a384-7b0e8578f9dc", + "userMandateId": "70f9e733-7297-4afe-8784-9f7c730de3c4", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" + }, + { + "id": "54232df1-b559-44bb-8a51-adbd82818d94", + "userMandateId": "412ba93d-2abe-4916-8a80-bec4672c0baf", + "roleId": "4564beaf-f07d-420f-a8af-d46dff0ba3b3" + } + ] + }, + "summary": { + "AccessRule": { + "recordCount": 96 + }, + "Mandate": { + "recordCount": 1 + }, + "Role": { + "recordCount": 4 + }, + "UserInDB": { + "recordCount": 2 + }, + "UserMandate": { + "recordCount": 2 + }, + "UserMandateRole": { + "recordCount": 2 + } + } +} \ No newline at end of file diff --git a/modules/datamodels/datamodelAudit.py b/modules/datamodels/datamodelAudit.py new file mode 100644 index 00000000..76c9ecfb --- /dev/null +++ b/modules/datamodels/datamodelAudit.py @@ -0,0 +1,208 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Audit Log Data Model for database-based audit logging. + +This model stores security-relevant audit events for GDPR compliance and security monitoring. + +GDPR-Relevant Events: +- User access: login, logout, failed login attempts +- Data access: create, read, update, delete operations on personal data +- Security events: password changes, token refresh, session management +- Key access: encryption/decryption of sensitive data +- GDPR actions: data export, data portability, account deletion +- Mandate/permission changes: user added/removed from mandates, role changes +""" + +from typing import Optional +from pydantic import BaseModel, Field +from enum import Enum +import uuid + +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared.attributeUtils import registerModelLabels + + +class AuditCategory(str, Enum): + """Categories for audit log entries""" + ACCESS = "access" # Login/logout events + KEY = "key" # Encryption key access + DATA = "data" # Data CRUD operations + SECURITY = "security" # Security-related events + GDPR = "gdpr" # GDPR-specific actions + PERMISSION = "permission" # Permission/role changes + SYSTEM = "system" # System-level events + + +class AuditAction(str, Enum): + """Actions for audit log entries""" + # Access actions + LOGIN = "login" + LOGIN_FAILED = "login_failed" + LOGOUT = "logout" + TOKEN_REFRESH = "token_refresh" + TOKEN_REVOKE = "token_revoke" + SESSION_EXPIRED = "session_expired" + + # Key actions + KEY_ENCODE = "encode" + KEY_DECODE = "decode" + KEY_ACCESS = "key_access" + + # Data actions + DATA_CREATE = "create" + DATA_READ = "read" + DATA_UPDATE = "update" + DATA_DELETE = "delete" + DATA_EXPORT = "export" + + # Security actions + PASSWORD_CHANGE = "password_change" + PASSWORD_RESET = "password_reset" + MFA_ENABLED = "mfa_enabled" + MFA_DISABLED = "mfa_disabled" + + # GDPR actions + GDPR_DATA_EXPORT = "gdpr_data_export" + GDPR_DATA_PORTABILITY = "gdpr_data_portability" + GDPR_ACCOUNT_DELETION = "gdpr_account_deletion" + GDPR_CONSENT_UPDATE = "gdpr_consent_update" + + # Permission actions + USER_ADDED_TO_MANDATE = "user_added_to_mandate" + USER_REMOVED_FROM_MANDATE = "user_removed_from_mandate" + ROLE_ASSIGNED = "role_assigned" + ROLE_REVOKED = "role_revoked" + FEATURE_ACCESS_GRANTED = "feature_access_granted" + FEATURE_ACCESS_REVOKED = "feature_access_revoked" + + # System actions + SYSTEM_STARTUP = "system_startup" + SYSTEM_SHUTDOWN = "system_shutdown" + CONFIG_CHANGE = "config_change" + + +class AuditLogEntry(BaseModel): + """ + Audit log entry for database storage. + + Stores all security-relevant events for compliance and monitoring. + Entries are immutable once created (append-only audit trail). + """ + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Unique identifier for the audit entry", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Timestamp + timestamp: float = Field( + default_factory=getUtcTimestamp, + description="UTC timestamp when the event occurred", + json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": True} + ) + + # Actor identification + userId: str = Field( + description="ID of the user who performed the action (or 'system' for system events)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + username: Optional[str] = Field( + default=None, + description="Username at the time of the event (for historical reference)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Context + mandateId: Optional[str] = Field( + default=None, + description="Mandate context (if applicable)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature instance context (if applicable)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Event classification + category: str = Field( + description="Event category (access, key, data, security, gdpr, permission, system)", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + action: str = Field( + description="Specific action performed", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} + ) + + # Event details + resourceType: Optional[str] = Field( + default=None, + description="Type of resource affected (e.g., 'User', 'ChatWorkflow', 'TrusteeContract')", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + resourceId: Optional[str] = Field( + default=None, + description="ID of the affected resource", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + details: Optional[str] = Field( + default=None, + description="Additional details about the event", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} + ) + + # Request metadata + ipAddress: Optional[str] = Field( + default=None, + description="IP address of the client", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + userAgent: Optional[str] = Field( + default=None, + description="User agent string from the request", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + + # Outcome + success: bool = Field( + default=True, + description="Whether the action was successful", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": True} + ) + + errorMessage: Optional[str] = Field( + default=None, + description="Error message if the action failed", + json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} + ) + + +# Register labels for internationalization +registerModelLabels( + "AuditLogEntry", + {"en": "Audit Log Entry", "de": "Audit-Log-Eintrag", "fr": "Entrée du journal d'audit"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "timestamp": {"en": "Timestamp", "de": "Zeitstempel", "fr": "Horodatage"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "username": {"en": "Username", "de": "Benutzername", "fr": "Nom d'utilisateur"}, + "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "de": "Feature-Instanz-ID", "fr": "ID de l'instance"}, + "category": {"en": "Category", "de": "Kategorie", "fr": "Catégorie"}, + "action": {"en": "Action", "de": "Aktion", "fr": "Action"}, + "resourceType": {"en": "Resource Type", "de": "Ressourcentyp", "fr": "Type de ressource"}, + "resourceId": {"en": "Resource ID", "de": "Ressourcen-ID", "fr": "ID de ressource"}, + "details": {"en": "Details", "de": "Details", "fr": "Détails"}, + "ipAddress": {"en": "IP Address", "de": "IP-Adresse", "fr": "Adresse IP"}, + "userAgent": {"en": "User Agent", "de": "User-Agent", "fr": "Agent utilisateur"}, + "success": {"en": "Success", "de": "Erfolgreich", "fr": "Succès"}, + "errorMessage": {"en": "Error Message", "de": "Fehlermeldung", "fr": "Message d'erreur"}, + }, +) diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChatbot.py similarity index 95% rename from modules/datamodels/datamodelChat.py rename to modules/datamodels/datamodelChatbot.py index 7860658c..45c5c4eb 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChatbot.py @@ -14,6 +14,12 @@ class ChatStat(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this stat belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this stat belongs to" + ) workflowId: Optional[str] = Field( None, description="Foreign key to workflow (for workflow stats)" ) @@ -33,6 +39,8 @@ registerModelLabels( {"en": "Chat Statistics", "fr": "Statistiques de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, @@ -49,6 +57,12 @@ class ChatLog(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this log belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this log belongs to" + ) workflowId: str = Field(description="Foreign key to workflow") message: str = Field(description="Log message") type: str = Field(description="Log type (info, warning, error, etc.)") @@ -79,6 +93,8 @@ registerModelLabels( {"en": "Chat Log", "fr": "Journal de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "message": {"en": "Message", "fr": "Message"}, "type": {"en": "Type", "fr": "Type"}, @@ -94,6 +110,12 @@ class ChatDocument(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this document belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to" + ) messageId: str = Field(description="Foreign key to message") fileId: str = Field(description="Foreign key to file") fileName: str = Field(description="Name of the file") @@ -112,6 +134,8 @@ registerModelLabels( {"en": "Chat Document", "fr": "Document de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "messageId": {"en": "Message ID", "fr": "ID du message"}, "fileId": {"en": "File ID", "fr": "ID du fichier"}, "fileName": {"en": "File Name", "fr": "Nom du fichier"}, @@ -200,6 +224,12 @@ class ChatMessage(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) + mandateId: str = Field( + description="ID of the mandate this message belongs to" + ) + featureInstanceId: str = Field( + description="ID of the feature instance this message belongs to" + ) workflowId: str = Field(description="Foreign key to workflow") parentMessageId: Optional[str] = Field( None, description="Parent message ID for threading" @@ -251,6 +281,8 @@ registerModelLabels( {"en": "Chat Message", "fr": "Message de chat"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, "documents": {"en": "Documents", "fr": "Documents"}, @@ -296,6 +328,7 @@ registerModelLabels( class ChatWorkflow(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this workflow belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) status: str = Field(default="running", description="Current status of the workflow", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, @@ -370,6 +403,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "status": {"en": "Status", "fr": "Statut"}, "name": {"en": "Name", "fr": "Nom"}, "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, @@ -993,6 +1027,7 @@ registerModelLabels( class AutomationDefinition(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="Mandate ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this automation belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) label: str = Field(description="User-friendly name", json_schema_extra={"frontend_type": "text", "frontend_required": True}) schedule: str = Field(description="Cron schedule pattern", json_schema_extra={"frontend_type": "select", "frontend_required": True, "frontend_options": [ {"value": "0 */4 * * *", "label": {"en": "Every 4 hours", "fr": "Toutes les 4 heures"}}, @@ -1013,6 +1048,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "label": {"en": "Label", "fr": "Libellé"}, "schedule": {"en": "Schedule", "fr": "Planification"}, "template": {"en": "Template", "fr": "Modèle"}, diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index 5deafa42..07880f3d 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -13,6 +13,7 @@ import base64 class FileItem(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: str = Field(description="ID of the feature instance this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileName: str = Field(description="Name of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) mimeType: str = Field(description="MIME type of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileHash: str = Field(description="Hash of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) @@ -25,6 +26,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "fileName": {"en": "fileName", "fr": "Nom de fichier"}, "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, diff --git a/modules/datamodels/datamodelMessaging.py b/modules/datamodels/datamodelMessaging.py index 2ec09c40..1c2206b7 100644 --- a/modules/datamodels/datamodelMessaging.py +++ b/modules/datamodels/datamodelMessaging.py @@ -45,6 +45,10 @@ class MessagingSubscription(BaseModel): description="ID of the mandate this subscription belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + featureInstanceId: str = Field( + description="ID of the feature instance this subscription belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) description: Optional[str] = Field( default=None, description="Description of the subscription", @@ -92,6 +96,7 @@ registerModelLabels( "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "subscriptionLabel": {"en": "Subscription Label", "fr": "Label d'abonnement"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "description": {"en": "Description", "fr": "Description"}, "isSystemSubscription": {"en": "System Subscription", "fr": "Abonnement système"}, "enabled": {"en": "Enabled", "fr": "Activé"}, @@ -110,6 +115,14 @@ class MessagingSubscriptionRegistration(BaseModel): description="Unique ID of the registration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + mandateId: str = Field( + description="ID of the mandate this registration belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureInstanceId: str = Field( + description="ID of the feature instance this registration belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) subscriptionId: str = Field( description="ID of the subscription this registration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True} @@ -161,6 +174,8 @@ registerModelLabels( {"en": "Messaging Registration", "fr": "Inscription à la messagerie"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "channel": {"en": "Channel", "fr": "Canal"}, @@ -179,6 +194,14 @@ class MessagingDelivery(BaseModel): description="Unique ID of the delivery", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) + mandateId: str = Field( + description="ID of the mandate this delivery belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) + featureInstanceId: str = Field( + description="ID of the feature instance this delivery belongs to", + json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} + ) subscriptionId: str = Field( description="ID of the subscription this delivery belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} @@ -239,6 +262,8 @@ registerModelLabels( {"en": "Messaging Delivery", "fr": "Livraison de messagerie"}, { "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "subscriptionId": {"en": "Subscription ID", "fr": "ID d'abonnement"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "channel": {"en": "Channel", "fr": "Canal"}, diff --git a/modules/datamodels/datamodelNeutralizer.py b/modules/datamodels/datamodelNeutralizer.py index 9d92bd60..e7b46c4d 100644 --- a/modules/datamodels/datamodelNeutralizer.py +++ b/modules/datamodels/datamodelNeutralizer.py @@ -11,6 +11,7 @@ from modules.shared.attributeUtils import registerModelLabels class DataNeutraliserConfig(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) userId: str = Field(description="ID of the user who created this configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) enabled: bool = Field(default=True, description="Whether data neutralization is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) namesToParse: str = Field(default="", description="Multiline list of names to parse for neutralization", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False}) @@ -22,6 +23,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "enabled": {"en": "Enabled", "fr": "Activé"}, "namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"}, @@ -33,6 +35,7 @@ registerModelLabels( class DataNeutralizerAttributes(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the attribute mapping (used as UID in neutralized files)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) userId: str = Field(description="ID of the user who created this attribute", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) originalText: str = Field(description="Original text that was neutralized", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) fileId: Optional[str] = Field(default=None, description="ID of the file this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) @@ -43,6 +46,7 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID"}, "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "originalText": {"en": "Original Text", "fr": "Texte original"}, "fileId": {"en": "File ID", "fr": "ID de fichier"}, diff --git a/modules/datamodels/datamodelRealEstate.py b/modules/datamodels/datamodelRealEstate.py index fa9e717e..31efbc07 100644 --- a/modules/datamodels/datamodelRealEstate.py +++ b/modules/datamodels/datamodelRealEstate.py @@ -123,6 +123,12 @@ class Dokument(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance this document belongs to", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Document label", frontend_type="text", @@ -207,6 +213,12 @@ class Land(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Country name (e.g. 'Schweiz')", frontend_type="text", @@ -251,6 +263,12 @@ class Kanton(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Canton name (e.g. 'Zürich')", frontend_type="text", @@ -302,6 +320,12 @@ class Gemeinde(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Municipality name (e.g. 'Zürich')", frontend_type="text", @@ -359,6 +383,12 @@ class Parzelle(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) # Grunddaten label: str = Field( @@ -579,6 +609,12 @@ class Projekt(BaseModel): frontend_readonly=True, frontend_required=False, ) + featureInstanceId: str = Field( + description="ID of the feature instance", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) label: str = Field( description="Project designation", frontend_type="text", @@ -643,6 +679,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "statusProzess": {"en": "Process Status", "fr": "Statut du processus", "de": "Prozessstatus"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) @@ -653,6 +690,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID", "de": "ID"}, "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) @@ -662,6 +700,8 @@ registerModelLabels( { "id": {"en": "ID", "fr": "ID", "de": "ID"}, "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance", "de": "Feature-Instanz-ID"}, }, ) diff --git a/modules/datamodels/datamodelTrustee.py b/modules/datamodels/datamodelTrustee.py index 21a3d3cc..e2d9b261 100644 --- a/modules/datamodels/datamodelTrustee.py +++ b/modules/datamodels/datamodelTrustee.py @@ -44,6 +44,15 @@ class TrusteeOrganisation(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector: # _createdAt, _modifiedAt, _createdBy, _modifiedBy @@ -56,6 +65,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "enabled": {"en": "Enabled", "fr": "Activé", "de": "Aktiviert"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -87,6 +97,15 @@ class TrusteeRole(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -97,6 +116,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID", "de": "ID"}, "desc": {"en": "Description", "fr": "Description", "de": "Beschreibung"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -159,6 +179,15 @@ class TrusteeAccess(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -172,6 +201,7 @@ registerModelLabels( "userId": {"en": "User", "fr": "Utilisateur", "de": "Benutzer"}, "contractId": {"en": "Contract (optional)", "fr": "Contrat (optionnel)", "de": "Vertrag (optional)"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -222,6 +252,15 @@ class TrusteeContract(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -234,6 +273,7 @@ registerModelLabels( "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, "enabled": {"en": "Enabled", "fr": "Activé", "de": "Aktiviert"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -309,6 +349,15 @@ class TrusteeDocument(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -323,6 +372,7 @@ registerModelLabels( "documentName": {"en": "Document Name", "fr": "Nom du document", "de": "Dokumentname"}, "documentMimeType": {"en": "MIME Type", "fr": "Type MIME", "de": "MIME-Typ"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -477,6 +527,15 @@ class TrusteePosition(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -499,6 +558,7 @@ registerModelLabels( "vatPercentage": {"en": "VAT Percentage", "fr": "Pourcentage TVA", "de": "MwSt-Prozentsatz"}, "vatAmount": {"en": "VAT Amount", "fr": "Montant TVA", "de": "MwSt-Betrag"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) @@ -562,6 +622,15 @@ class TrusteePositionDocument(BaseModel): "frontend_required": False } ) + featureInstanceId: Optional[str] = Field( + default=None, + description="Feature Instance ID for instance-level isolation", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False + } + ) # System attributes are automatically set by DatabaseConnector @@ -575,5 +644,6 @@ registerModelLabels( "documentId": {"en": "Document", "fr": "Document", "de": "Dokument"}, "positionId": {"en": "Position", "fr": "Position", "de": "Position"}, "mandateId": {"en": "Mandate", "fr": "Mandat", "de": "Mandat"}, + "featureInstanceId": {"en": "Feature Instance", "fr": "Instance de fonctionnalité", "de": "Feature-Instanz"}, }, ) diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py index bb1ed9ca..86f4bb1d 100644 --- a/modules/datamodels/datamodelVoice.py +++ b/modules/datamodels/datamodelVoice.py @@ -12,6 +12,7 @@ class VoiceSettings(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) mandateId: str = Field(description="ID of the mandate these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + featureInstanceId: str = Field(description="ID of the feature instance these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) @@ -28,6 +29,7 @@ registerModelLabels( "id": {"en": "ID", "fr": "ID"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py index b884382c..19117fce 100644 --- a/modules/datamodels/datamodelWorkflow.py +++ b/modules/datamodels/datamodelWorkflow.py @@ -14,7 +14,7 @@ from modules.datamodels.datamodelDocref import DocumentReferenceList # Forward references for circular imports (use string annotations) if TYPE_CHECKING: - from modules.datamodels.datamodelChat import ChatDocument, ActionResult + from modules.datamodels.datamodelChatbot import ChatDocument, ActionResult from modules.datamodels.datamodelExtraction import ExtractionOptions diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index 8bac1fd5..1ca90d51 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 43503339..a5222966 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -13,7 +13,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -335,7 +335,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChat import ChatLog + from modules.datamodels.datamodelChatbot import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 37c4a3cd..0d386aac 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -23,7 +23,7 @@ from modules.datamodels.datamodelRealEstate import ( Land, ) from modules.services import getInterface as getServices -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector logger = logging.getLogger(__name__) diff --git a/modules/features/workflow/mainWorkflow.py b/modules/features/workflow/mainWorkflow.py index 70a2e9aa..ab92510c 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/features/workflow/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/features/workflow/subAutomationUtils.py b/modules/features/workflow/subAutomationUtils.py index 60993b62..906c9caa 100644 --- a/modules/features/workflow/subAutomationUtils.py +++ b/modules/features/workflow/subAutomationUtils.py @@ -3,7 +3,7 @@ """ Utility functions for automation feature. -Moved from interfaces/interfaceDbChatObjects.py. +Moved from interfaces/interfaceDbChatbot.py. """ import json diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatbot.py similarity index 95% rename from modules/interfaces/interfaceDbChatObjects.py rename to modules/interfaces/interfaceDbChatbot.py index 10b202da..9ded2dd8 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( ChatDocument, ChatStat, ChatLog, @@ -178,18 +178,20 @@ class ChatObjects: Uses the JSON connector for data access with added language support. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Chat Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ # Initialize variables self.currentUser = currentUser # Store User object directly self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # RBAC interface # Initialize services @@ -200,7 +202,7 @@ class ChatObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) # ===== Generic Utility Methods ===== @@ -263,17 +265,19 @@ class ChatObjects: def _initializeServices(self): pass - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser # Store User object directly self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -603,10 +607,12 @@ class ChatObjects: If pagination is None: List[Dict[str, Any]] If pagination is provided: PaginatedResult with items and metadata """ - # Use RBAC filtering + # Use RBAC filtering with featureInstanceId for instance-level isolation filteredWorkflows = getRecordsetWithRBAC(self.db, ChatWorkflow, - self.currentUser + self.currentUser, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) # If no pagination requested, return all items (no sorting - frontend handles it) @@ -638,11 +644,13 @@ class ChatObjects: def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: """Returns a workflow by ID if user has access.""" - # Use RBAC filtering + # Use RBAC filtering with featureInstanceId for instance-level isolation workflows = getRecordsetWithRBAC(self.db, ChatWorkflow, self.currentUser, - recordFilter={"id": workflowId} + recordFilter={"id": workflowId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId ) if not workflows: @@ -689,6 +697,12 @@ class ChatObjects: if "lastActivity" not in workflowData: workflowData["lastActivity"] = currentTime + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in workflowData or not workflowData["mandateId"]: + workflowData["mandateId"] = self.mandateId + if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: + workflowData["featureInstanceId"] = self.featureInstanceId + # Use generic field separation based on ChatWorkflow model simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) @@ -1009,6 +1023,12 @@ class ChatObjects: if "actionNumber" not in messageData: messageData["actionNumber"] = workflow.currentAction + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in messageData or not messageData["mandateId"]: + messageData["mandateId"] = self.mandateId + if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: + messageData["featureInstanceId"] = self.featureInstanceId + # Use generic field separation based on ChatMessage model simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) @@ -1303,6 +1323,12 @@ class ChatObjects: def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: """Creates a document for a message in normalized table.""" try: + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in documentData or not documentData["mandateId"]: + documentData["mandateId"] = self.mandateId + if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: + documentData["featureInstanceId"] = self.featureInstanceId + # Validate and normalize document data to dict document = ChatDocument(**documentData) logger.debug(f"Creating document in database: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") @@ -1422,6 +1448,12 @@ class ChatObjects: if "timestamp" not in logData: logData["timestamp"] = getUtcTimestamp() + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in logData or not logData["mandateId"]: + logData["mandateId"] = self.mandateId + if "featureInstanceId" not in logData or not logData["featureInstanceId"]: + logData["featureInstanceId"] = self.featureInstanceId + # Add status information if not present if "status" not in logData and "type" in logData: if logData["type"] == "error": @@ -1508,6 +1540,12 @@ class ChatObjects: if "workflowId" not in statData: raise ValueError("workflowId is required in statData") + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in statData or not statData["mandateId"]: + statData["mandateId"] = self.mandateId + if "featureInstanceId" not in statData or not statData["featureInstanceId"]: + statData["featureInstanceId"] = self.featureInstanceId + # Validate the stat data against ChatStat model stat = ChatStat(**statData) @@ -1768,9 +1806,11 @@ class ChatObjects: if "id" not in automationData or not automationData["id"]: automationData["id"] = str(uuid.uuid4()) - # Ensure mandateId is set + # Ensure mandateId and featureInstanceId are set for proper data isolation if "mandateId" not in automationData: automationData["mandateId"] = self.mandateId + if "featureInstanceId" not in automationData: + automationData["featureInstanceId"] = self.featureInstanceId # Ensure database connector has correct userId context # The connector should have been initialized with userId, but ensure it's updated @@ -1894,7 +1934,7 @@ class ChatObjects: logger.error(f"Error notifying automation change: {str(e)}") -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ChatObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': """ Returns a ChatObjects instance for the current user. Handles initialization of database and records. @@ -1902,20 +1942,22 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ if not currentUser: raise ValueError("Invalid user context: user is required") effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - # Create context key - contextKey = f"{effectiveMandateId}_{currentUser.id}" + # Create context key including featureInstanceId for proper isolation + contextKey = f"{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" # Create new instance if not exists if contextKey not in _chatInterfaces: - _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId) + _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _chatInterfaces[contextKey] diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbComponentObjects.py index 07cf235c..72d65839 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbComponentObjects.py @@ -76,12 +76,13 @@ class ComponentObjects: # Initialize standard records if needed self._initRecords() - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ if not currentUser: logger.info("Initializing interface without user context") @@ -91,6 +92,7 @@ class ComponentObjects: self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -986,12 +988,14 @@ class ComponentObjects: fileSize = len(content) fileHash = hashlib.sha256(content).hexdigest() - # Use mandateId from context + # Use mandateId and featureInstanceId from context for proper data isolation mandateId = self.mandateId + featureInstanceId = self.featureInstanceId # Create FileItem instance fileItem = FileItem( mandateId=mandateId, + featureInstanceId=featureInstanceId, fileName=uniqueName, mimeType=mimeType, fileSize=fileSize, @@ -1327,9 +1331,11 @@ class ComponentObjects: if "userId" not in settingsData: settingsData["userId"] = self.userId - # Ensure mandateId is set from context + # Ensure mandateId and featureInstanceId are set from context if "mandateId" not in settingsData: settingsData["mandateId"] = self.mandateId + if "featureInstanceId" not in settingsData: + settingsData["featureInstanceId"] = self.featureInstanceId # Check if settings already exist for this user existingSettings = self.getVoiceSettings(settingsData["userId"]) @@ -1501,9 +1507,11 @@ class ComponentObjects: if not all(c.isalpha() or c == "_" for c in subscriptionId): raise ValueError("subscriptionId must contain only letters and underscores") - # Set mandateId from context + # Set mandateId and featureInstanceId from context for proper data isolation if "mandateId" not in subscriptionData: subscriptionData["mandateId"] = self.mandateId + if "featureInstanceId" not in subscriptionData: + subscriptionData["featureInstanceId"] = self.featureInstanceId createdRecord = self.db.recordCreate(MessagingSubscription, subscriptionData) if not createdRecord or not createdRecord.get("id"): @@ -1605,6 +1613,12 @@ class ComponentObjects: if "userId" not in registrationData: registrationData["userId"] = self.userId + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in registrationData: + registrationData["mandateId"] = self.mandateId + if "featureInstanceId" not in registrationData: + registrationData["featureInstanceId"] = self.featureInstanceId + createdRecord = self.db.recordCreate(MessagingSubscriptionRegistration, registrationData) if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create registration record") @@ -1679,6 +1693,13 @@ class ComponentObjects: def createDelivery(self, delivery: MessagingDelivery) -> Dict[str, Any]: """Creates a new delivery record.""" deliveryData = delivery.model_dump() if isinstance(delivery, MessagingDelivery) else delivery + + # Set mandateId and featureInstanceId from context for proper data isolation + if "mandateId" not in deliveryData or not deliveryData["mandateId"]: + deliveryData["mandateId"] = self.mandateId + if "featureInstanceId" not in deliveryData or not deliveryData["featureInstanceId"]: + deliveryData["featureInstanceId"] = self.featureInstanceId + createdRecord = self.db.recordCreate(MessagingDelivery, deliveryData) if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create delivery record") @@ -1748,7 +1769,7 @@ class ComponentObjects: return MessagingDelivery(**filteredDeliveries[0]) if filteredDeliveries else None -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None) -> 'ComponentObjects': +def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ComponentObjects': """ Returns a ComponentObjects instance. If currentUser is provided, initializes with user context. @@ -1757,8 +1778,10 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None # Create new instance if not exists if "default" not in _instancesManagement: @@ -1767,7 +1790,7 @@ def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = interface = _instancesManagement["default"] if currentUser: - interface.setUserContext(currentUser, mandateId=effectiveMandateId) + interface.setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: logger.info("Returning interface without user context") diff --git a/modules/interfaces/interfaceDbRealEstateObjects.py b/modules/interfaces/interfaceDbRealEstate.py similarity index 93% rename from modules/interfaces/interfaceDbRealEstateObjects.py rename to modules/interfaces/interfaceDbRealEstate.py index d475b8db..179ec6dd 100644 --- a/modules/interfaces/interfaceDbRealEstateObjects.py +++ b/modules/interfaces/interfaceDbRealEstate.py @@ -39,17 +39,19 @@ class RealEstateObjects: Handles CRUD operations on Real Estate entities. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Real Estate Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # RBAC interface # Initialize database @@ -57,7 +59,7 @@ class RealEstateObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) def _initializeDatabase(self): """Initialize PostgreSQL database connection.""" @@ -107,17 +109,19 @@ class RealEstateObjects: logger.warning(f"Error ensuring supporting tables exist: {e}") # Don't raise - tables will be created on-demand anyway - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -145,9 +149,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Projekt, "create"): raise PermissionError(f"User {self.userId} cannot create projects") - # Ensure mandateId is set + # Ensure mandateId and featureInstanceId are set for proper data isolation if not projekt.mandateId: projekt.mandateId = self.mandateId + if not projekt.featureInstanceId: + projekt.featureInstanceId = self.featureInstanceId # Save to database - use mode='json' to ensure nested Pydantic models are serialized self.db.recordCreate(Projekt, projekt.model_dump(mode='json')) @@ -231,8 +237,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Parzelle, "create"): raise PermissionError(f"User {self.userId} cannot create plots") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not parzelle.mandateId: parzelle.mandateId = self.mandateId + if not parzelle.featureInstanceId: + parzelle.featureInstanceId = self.featureInstanceId # Use mode='json' to ensure nested Pydantic models (like GeoPolylinie) are serialized self.db.recordCreate(Parzelle, parzelle.model_dump(mode='json')) @@ -438,8 +447,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Dokument, "create"): raise PermissionError(f"User {self.userId} cannot create documents") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not dokument.mandateId: dokument.mandateId = self.mandateId + if not dokument.featureInstanceId: + dokument.featureInstanceId = self.featureInstanceId self.db.recordCreate(Dokument, dokument.model_dump()) @@ -504,8 +516,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Gemeinde, "create"): raise PermissionError(f"User {self.userId} cannot create municipalities") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not gemeinde.mandateId: gemeinde.mandateId = self.mandateId + if not gemeinde.featureInstanceId: + gemeinde.featureInstanceId = self.featureInstanceId self.db.recordCreate(Gemeinde, gemeinde.model_dump()) @@ -570,8 +585,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Kanton, "create"): raise PermissionError(f"User {self.userId} cannot create cantons") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not kanton.mandateId: kanton.mandateId = self.mandateId + if not kanton.featureInstanceId: + kanton.featureInstanceId = self.featureInstanceId self.db.recordCreate(Kanton, kanton.model_dump()) @@ -636,8 +654,11 @@ class RealEstateObjects: if not self.checkRbacPermission(Land, "create"): raise PermissionError(f"User {self.userId} cannot create countries") + # Ensure mandateId and featureInstanceId are set for proper data isolation if not land.mandateId: land.mandateId = self.mandateId + if not land.featureInstanceId: + land.featureInstanceId = self.featureInstanceId self.db.recordCreate(Land, land.model_dump()) @@ -792,7 +813,7 @@ class RealEstateObjects: raise -def getInterface(currentUser: User, mandateId: Optional[str] = None) -> RealEstateObjects: +def getInterface(currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> RealEstateObjects: """ Factory function to get or create a Real Estate interface instance for a user. Uses singleton pattern per user. @@ -800,16 +821,19 @@ def getInterface(currentUser: User, mandateId: Optional[str] = None) -> RealEsta Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - userKey = f"{currentUser.id}_{effectiveMandateId}" + # Include featureInstanceId in key for proper isolation + userKey = f"{currentUser.id}_{effectiveMandateId}_{effectiveFeatureInstanceId}" if userKey not in _realEstateInterfaces: - _realEstateInterfaces[userKey] = RealEstateObjects(currentUser, mandateId=effectiveMandateId) + _realEstateInterfaces[userKey] = RealEstateObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _realEstateInterfaces[userKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _realEstateInterfaces[userKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _realEstateInterfaces[userKey] diff --git a/modules/interfaces/interfaceDbTrusteeObjects.py b/modules/interfaces/interfaceDbTrustee.py similarity index 96% rename from modules/interfaces/interfaceDbTrusteeObjects.py rename to modules/interfaces/interfaceDbTrustee.py index edb085fa..56ce19b3 100644 --- a/modules/interfaces/interfaceDbTrusteeObjects.py +++ b/modules/interfaces/interfaceDbTrustee.py @@ -33,12 +33,13 @@ logger = logging.getLogger(__name__) _trusteeInterfaces = {} -def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = None) -> "TrusteeObjects": +def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = None, featureInstanceId: Optional[str] = None) -> "TrusteeObjects": """Get or create a TrusteeObjects instance for the given user context. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). """ global _trusteeInterfaces @@ -46,14 +47,16 @@ def getInterface(currentUser: User, mandateId: Optional[Union[str, uuid.UUID]] = raise ValueError("Valid user context required") effectiveMandateId = str(mandateId) if mandateId else None + effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - cacheKey = f"{currentUser.id}_{effectiveMandateId}" + # Include featureInstanceId in cache key for proper isolation + cacheKey = f"{currentUser.id}_{effectiveMandateId}_{effectiveFeatureInstanceId}" if cacheKey not in _trusteeInterfaces: - _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser, mandateId=effectiveMandateId) + _trusteeInterfaces[cacheKey] = TrusteeObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) else: # Update user context if needed - _trusteeInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId) + _trusteeInterfaces[cacheKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) return _trusteeInterfaces[cacheKey] @@ -64,17 +67,19 @@ class TrusteeObjects: Manages trustee organisations, roles, access, contracts, documents, and positions. """ - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None): + def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Initializes the Trustee Interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ self.currentUser = currentUser self.userId = currentUser.id if currentUser else None # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId self.rbac = None # Initialize database @@ -82,14 +87,15 @@ class TrusteeObjects: # Set user context if provided if currentUser: - self.setUserContext(currentUser, mandateId=mandateId) + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None): + def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """Sets the user context for the interface. Args: currentUser: The authenticated user mandateId: The mandate ID from RequestContext (X-Mandate-Id header) + featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) """ if not currentUser: logger.info("Initializing interface without user context") @@ -99,6 +105,7 @@ class TrusteeObjects: self.userId = currentUser.id # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId + self.featureInstanceId = featureInstanceId if not self.userId: raise ValueError("Invalid user context: id is required") @@ -204,8 +211,10 @@ class TrusteeObjects: logger.warning(f"User {self.userId} lacks permission to create organisation") return None - # Set mandateId from current user + # Set mandateId and featureInstanceId from context for proper data isolation data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId # Validate ID format (alphanumeric, hyphens, underscores, 3-50 chars) orgId = data.get("id", "") @@ -307,6 +316,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId roleId = data.get("id", "") if not roleId: @@ -414,6 +425,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid accessId = data.get("id") or str(uuid.uuid4()) @@ -603,6 +616,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid contractId = data.get("id") or str(uuid.uuid4()) @@ -729,6 +744,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid documentId = data.get("id") or str(uuid.uuid4()) @@ -879,6 +896,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId # Calculate VAT amount if not provided if "vatAmount" not in data or data.get("vatAmount") == 0: @@ -1028,6 +1047,8 @@ class TrusteeObjects: return None data["mandateId"] = self.mandateId + if "featureInstanceId" not in data: + data["featureInstanceId"] = self.featureInstanceId import uuid linkId = data.get("id") or str(uuid.uuid4()) diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 3b5515df..071f8673 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -13,9 +13,9 @@ import logging import json # Import interfaces and models -from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface +from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow +from modules.datamodels.datamodelChatbot import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.features.workflow import executeAutomation diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index a118869a..bf3fbc73 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -472,12 +472,15 @@ async def addUserToMandate( roleIds=data.roleIds ) - # 8. Audit - audit_logger.logSecurityEvent( + # 8. Audit - Log permission change with IP address + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, action="user_added_to_mandate", - details=f"targetUser={data.targetUserId}, roles={data.roleIds}" + targetUserId=data.targetUserId, + details=f"Roles assigned: {data.roleIds}", + resourceType="UserMandate", + resourceId=str(userMandate.id) ) logger.info( @@ -557,12 +560,14 @@ async def removeUserFromMandate( # Delete UserMandate (CASCADE will delete UserMandateRole entries) rootInterface.deleteUserMandate(targetUserId, mandateId) - # Audit - audit_logger.logSecurityEvent( + # Audit - Log permission change + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, action="user_removed_from_mandate", - details=f"targetUser={targetUserId}" + targetUserId=targetUserId, + details="User removed from mandate", + resourceType="UserMandate" ) logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {mandateId}") @@ -657,12 +662,15 @@ async def updateUserRolesInMandate( for roleId in roleIds: rootInterface.addRoleToUserMandate(str(membership.id), roleId) - # Audit - audit_logger.logSecurityEvent( + # Audit - Log role assignment change + audit_logger.logPermissionChange( userId=str(context.user.id), mandateId=mandateId, - action="user_roles_updated_in_mandate", - details=f"targetUser={targetUserId}, newRoles={roleIds}" + action="role_assigned", + targetUserId=targetUserId, + details=f"New roles: {roleIds}", + resourceType="UserMandateRole", + resourceId=str(membership.id) ) logger.info( diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index ae6ab70a..226f3606 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -360,7 +360,9 @@ async def reset_user_password( userId=str(context.user.id), mandateId=str(context.mandateId) if context.mandateId else "system", action="password_reset", - details=f"Reset password for user {userId}" + details=f"Reset password for user {userId}", + ipAddress=request.client.host if request.client else None, + success=True ) except Exception: pass @@ -439,7 +441,9 @@ async def change_password( userId=str(context.user.id), mandateId=str(context.mandateId) if context.mandateId else "system", action="password_change", - details="User changed their own password" + details="User changed their own password", + ipAddress=request.client.host if request.client else None, + success=True ) except Exception: pass diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index f2955b61..d6f53d84 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -13,10 +13,10 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Reques from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions from modules.features.workflow import chatStart, chatStop @@ -32,7 +32,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Workflow start endpoint @router.post("/start", response_model=ChatWorkflow) diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index 0505a752..20b90876 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -18,11 +18,11 @@ from modules.shared.timeUtils import parseTimestamp from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature @@ -43,7 +43,7 @@ router = APIRouter( ) def _getServiceChat(context: RequestContext): - return interfaceDbChatObjects.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + return interfaceDbChatbot.getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) # Chatbot streaming endpoint (SSE) @router.post("/start/stream") diff --git a/modules/routes/routeFeatureRealEstate.py b/modules/routes/routeFeatureRealEstate.py index fe7544de..7e130c1b 100644 --- a/modules/routes/routeFeatureRealEstate.py +++ b/modules/routes/routeFeatureRealEstate.py @@ -26,7 +26,7 @@ from modules.datamodels.datamodelRealEstate import ( ) # Import interfaces -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbRealEstate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands from modules.features.realEstate.mainRealEstate import ( diff --git a/modules/routes/routeFeatureTrustee.py b/modules/routes/routeFeatureTrustee.py index 69fd5918..ad842db8 100644 --- a/modules/routes/routeFeatureTrustee.py +++ b/modules/routes/routeFeatureTrustee.py @@ -3,6 +3,10 @@ """ Routes for Trustee feature data management. Implements CRUD operations for organisations, roles, access, contracts, documents, and positions. + +URL Structure: /api/trustee/{instanceId}/{entity} +- instanceId is the FeatureInstance ID (required for all operations) +- This ensures proper multi-tenant isolation at the URL level """ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Query, Response @@ -14,7 +18,9 @@ import json import io from modules.auth import limiter, getRequestContext, RequestContext -from modules.interfaces.interfaceDbTrusteeObjects import getInterface +from modules.interfaces.interfaceDbTrustee import getInterface +from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface from modules.datamodels.datamodelTrustee import ( TrusteeOrganisation, TrusteeRole, @@ -59,22 +65,72 @@ def _parsePagination(pagination: Optional[str]) -> Optional[PaginationParams]: return None +async def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: + """ + Validate that the user has access to the feature instance. + Returns the mandateId for the instance. + + Args: + instanceId: The FeatureInstance ID from URL + context: The request context with user info + + Returns: + mandateId of the instance + + Raises: + HTTPException 404 if instance not found + HTTPException 403 if user doesn't have access + """ + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=404, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify it's a trustee instance + if instance.featureCode != "trustee": + raise HTTPException( + status_code=400, + detail=f"Instance '{instanceId}' is not a trustee instance" + ) + + # Verify user has access to this instance + if not context.isSysAdmin: + # Check if user has FeatureAccess for this instance + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + hasAccess = any( + str(fa.featureInstanceId) == instanceId and fa.enabled + for fa in featureAccesses + ) + if not hasAccess: + raise HTTPException( + status_code=403, + detail=f"Access denied to feature instance '{instanceId}'" + ) + + return str(instance.mandateId) + + # ===== Organisation Routes ===== -@router.get("/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) +@router.get("/{instanceId}/organisations", response_model=PaginatedResponse[TrusteeOrganisation]) @limiter.limit("30/minute") async def getOrganisations( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeOrganisation]: - """Get all organisations with optional pagination.""" - logger = logging.getLogger(__name__) - logger.debug(f"getOrganisations called for user {context.user.id}, mandateId: {context.mandateId}") + """Get all organisations for a feature instance with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllOrganisations(paginationParams) - logger.debug(f"getOrganisations returned {len(result.items)} items") if paginationParams: return PaginatedResponse( @@ -91,46 +147,55 @@ async def getOrganisations( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/organisations/{orgId}", response_model=TrusteeOrganisation) +@router.get("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("30/minute") async def getOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Get a single organisation by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) org = interface.getOrganisation(orgId) if not org: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") return org -@router.post("/organisations", response_model=TrusteeOrganisation, status_code=201) +@router.post("/{instanceId}/organisations", response_model=TrusteeOrganisation, status_code=201) @limiter.limit("10/minute") async def createOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeOrganisation = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Create a new organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createOrganisation(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create organisation") return result -@router.put("/organisations/{orgId}", response_model=TrusteeOrganisation) +@router.put("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @limiter.limit("10/minute") async def updateOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), data: TrusteeOrganisation = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeOrganisation: """Update an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -141,15 +206,18 @@ async def updateOrganisation( return result -@router.delete("/organisations/{orgId}") +@router.delete("/{instanceId}/organisations/{orgId}") @limiter.limit("10/minute") async def deleteOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(..., description="Organisation ID"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getOrganisation(orgId) if not existing: raise HTTPException(status_code=404, detail=f"Organisation {orgId} not found") @@ -162,16 +230,19 @@ async def deleteOrganisation( # ===== Role Routes ===== -@router.get("/roles", response_model=PaginatedResponse[TrusteeRole]) +@router.get("/{instanceId}/roles", response_model=PaginatedResponse[TrusteeRole]) @limiter.limit("30/minute") async def getRoles( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeRole]: """Get all roles with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllRoles(paginationParams) if paginationParams: @@ -189,46 +260,55 @@ async def getRoles( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/roles/{roleId}", response_model=TrusteeRole) +@router.get("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("30/minute") async def getRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(..., description="Role ID"), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Get a single role by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) role = interface.getRole(roleId) if not role: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") return role -@router.post("/roles", response_model=TrusteeRole, status_code=201) +@router.post("/{instanceId}/roles", response_model=TrusteeRole, status_code=201) @limiter.limit("10/minute") async def createRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeRole = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Create a new role (sysadmin only).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createRole(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create role") return result -@router.put("/roles/{roleId}", response_model=TrusteeRole) +@router.put("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @limiter.limit("10/minute") async def updateRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), data: TrusteeRole = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeRole: """Update a role (sysadmin only).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -239,15 +319,18 @@ async def updateRole( return result -@router.delete("/roles/{roleId}") +@router.delete("/{instanceId}/roles/{roleId}") @limiter.limit("10/minute") async def deleteRole( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), roleId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a role (sysadmin only, fails if in use).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getRole(roleId) if not existing: raise HTTPException(status_code=404, detail=f"Role {roleId} not found") @@ -260,16 +343,19 @@ async def deleteRole( # ===== Access Routes ===== -@router.get("/access", response_model=PaginatedResponse[TrusteeAccess]) +@router.get("/{instanceId}/access", response_model=PaginatedResponse[TrusteeAccess]) @limiter.limit("30/minute") async def getAllAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeAccess]: """Get all access records with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllAccess(paginationParams) if paginationParams: @@ -287,70 +373,85 @@ async def getAllAccess( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/access/{accessId}", response_model=TrusteeAccess) +@router.get("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("30/minute") async def getAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Get a single access record by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) access = interface.getAccess(accessId) if not access: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") return access -@router.get("/access/organisation/{orgId}", response_model=List[TrusteeAccess]) +@router.get("/{instanceId}/access/organisation/{orgId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") async def getAccessByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getAccessByOrganisation(orgId) -@router.get("/access/user/{userId}", response_model=List[TrusteeAccess]) +@router.get("/{instanceId}/access/user/{userId}", response_model=List[TrusteeAccess]) @limiter.limit("30/minute") async def getAccessByUser( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), userId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeAccess]: """Get all access records for a user.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getAccessByUser(userId) -@router.post("/access", response_model=TrusteeAccess, status_code=201) +@router.post("/{instanceId}/access", response_model=TrusteeAccess, status_code=201) @limiter.limit("10/minute") async def createAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeAccess = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Create a new access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createAccess(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create access") return result -@router.put("/access/{accessId}", response_model=TrusteeAccess) +@router.put("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @limiter.limit("10/minute") async def updateAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), data: TrusteeAccess = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeAccess: """Update an access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -361,15 +462,18 @@ async def updateAccess( return result -@router.delete("/access/{accessId}") +@router.delete("/{instanceId}/access/{accessId}") @limiter.limit("10/minute") async def deleteAccess( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), accessId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete an access record.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getAccess(accessId) if not existing: raise HTTPException(status_code=404, detail=f"Access {accessId} not found") @@ -382,16 +486,19 @@ async def deleteAccess( # ===== Contract Routes ===== -@router.get("/contracts", response_model=PaginatedResponse[TrusteeContract]) +@router.get("/{instanceId}/contracts", response_model=PaginatedResponse[TrusteeContract]) @limiter.limit("30/minute") async def getContracts( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeContract]: """Get all contracts with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllContracts(paginationParams) if paginationParams: @@ -409,58 +516,70 @@ async def getContracts( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/contracts/{contractId}", response_model=TrusteeContract) +@router.get("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("30/minute") async def getContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Get a single contract by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) contract = interface.getContract(contractId) if not contract: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") return contract -@router.get("/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) +@router.get("/{instanceId}/contracts/organisation/{orgId}", response_model=List[TrusteeContract]) @limiter.limit("30/minute") async def getContractsByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeContract]: """Get all contracts for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getContractsByOrganisation(orgId) -@router.post("/contracts", response_model=TrusteeContract, status_code=201) +@router.post("/{instanceId}/contracts", response_model=TrusteeContract, status_code=201) @limiter.limit("10/minute") async def createContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeContract = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Create a new contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createContract(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create contract") return result -@router.put("/contracts/{contractId}", response_model=TrusteeContract) +@router.put("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @limiter.limit("10/minute") async def updateContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), data: TrusteeContract = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeContract: """Update a contract (organisationId is immutable).""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -471,15 +590,18 @@ async def updateContract( return result -@router.delete("/contracts/{contractId}") +@router.delete("/{instanceId}/contracts/{contractId}") @limiter.limit("10/minute") async def deleteContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getContract(contractId) if not existing: raise HTTPException(status_code=404, detail=f"Contract {contractId} not found") @@ -492,16 +614,19 @@ async def deleteContract( # ===== Document Routes ===== -@router.get("/documents", response_model=PaginatedResponse[TrusteeDocument]) +@router.get("/{instanceId}/documents", response_model=PaginatedResponse[TrusteeDocument]) @limiter.limit("30/minute") async def getDocuments( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteeDocument]: """Get all documents (metadata only) with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllDocuments(paginationParams) if paginationParams: @@ -519,30 +644,36 @@ async def getDocuments( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/documents/{documentId}", response_model=TrusteeDocument) +@router.get("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("30/minute") async def getDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Get document metadata by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") return doc -@router.get("/documents/{documentId}/data") +@router.get("/{instanceId}/documents/{documentId}/data") @limiter.limit("10/minute") async def getDocumentData( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ): """Download document binary data.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) doc = interface.getDocument(documentId) if not doc: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -558,43 +689,52 @@ async def getDocumentData( ) -@router.get("/documents/contract/{contractId}", response_model=List[TrusteeDocument]) +@router.get("/{instanceId}/documents/contract/{contractId}", response_model=List[TrusteeDocument]) @limiter.limit("30/minute") async def getDocumentsByContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteeDocument]: """Get all documents for a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getDocumentsByContract(contractId) -@router.post("/documents", response_model=TrusteeDocument, status_code=201) +@router.post("/{instanceId}/documents", response_model=TrusteeDocument, status_code=201) @limiter.limit("10/minute") async def createDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteeDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Create a new document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create document") return result -@router.put("/documents/{documentId}", response_model=TrusteeDocument) +@router.put("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @limiter.limit("10/minute") async def updateDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), data: TrusteeDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteeDocument: """Update document metadata.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -605,15 +745,18 @@ async def updateDocument( return result -@router.delete("/documents/{documentId}") +@router.delete("/{instanceId}/documents/{documentId}") @limiter.limit("10/minute") async def deleteDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getDocument(documentId) if not existing: raise HTTPException(status_code=404, detail=f"Document {documentId} not found") @@ -626,16 +769,19 @@ async def deleteDocument( # ===== Position Routes ===== -@router.get("/positions", response_model=PaginatedResponse[TrusteePosition]) +@router.get("/{instanceId}/positions", response_model=PaginatedResponse[TrusteePosition]) @limiter.limit("30/minute") async def getPositions( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePosition]: """Get all positions with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(paginationParams) if paginationParams: @@ -653,70 +799,85 @@ async def getPositions( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/positions/{positionId}", response_model=TrusteePosition) +@router.get("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("30/minute") async def getPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Get a single position by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) position = interface.getPosition(positionId) if not position: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") return position -@router.get("/positions/contract/{contractId}", response_model=List[TrusteePosition]) +@router.get("/{instanceId}/positions/contract/{contractId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") async def getPositionsByContract( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), contractId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for a contract.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsByContract(contractId) -@router.get("/positions/organisation/{orgId}", response_model=List[TrusteePosition]) +@router.get("/{instanceId}/positions/organisation/{orgId}", response_model=List[TrusteePosition]) @limiter.limit("30/minute") async def getPositionsByOrganisation( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), orgId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePosition]: """Get all positions for an organisation.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsByOrganisation(orgId) -@router.post("/positions", response_model=TrusteePosition, status_code=201) +@router.post("/{instanceId}/positions", response_model=TrusteePosition, status_code=201) @limiter.limit("10/minute") async def createPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePosition = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Create a new position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createPosition(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create position") return result -@router.put("/positions/{positionId}", response_model=TrusteePosition) +@router.put("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) @limiter.limit("10/minute") async def updatePosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), data: TrusteePosition = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePosition: """Update a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -727,15 +888,18 @@ async def updatePosition( return result -@router.delete("/positions/{positionId}") +@router.delete("/{instanceId}/positions/{positionId}") @limiter.limit("10/minute") async def deletePosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPosition(positionId) if not existing: raise HTTPException(status_code=404, detail=f"Position {positionId} not found") @@ -748,16 +912,19 @@ async def deletePosition( # ===== Position-Document Link Routes ===== -@router.get("/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents", response_model=PaginatedResponse[TrusteePositionDocument]) @limiter.limit("30/minute") async def getPositionDocuments( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), pagination: Optional[str] = Query(None), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[TrusteePositionDocument]: """Get all position-document links with optional pagination.""" + mandateId = await _validateInstanceAccess(instanceId, context) + paginationParams = _parsePagination(pagination) - interface = getInterface(context.user, mandateId=context.mandateId) + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositionDocuments(paginationParams) if paginationParams: @@ -775,69 +942,84 @@ async def getPositionDocuments( return PaginatedResponse(items=result.items, pagination=None) -@router.get("/position-documents/{linkId}", response_model=TrusteePositionDocument) +@router.get("/{instanceId}/position-documents/{linkId}", response_model=TrusteePositionDocument) @limiter.limit("30/minute") async def getPositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Get a single position-document link by ID.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) link = interface.getPositionDocument(linkId) if not link: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") return link -@router.get("/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents/position/{positionId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") async def getDocumentsForPosition( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), positionId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all document links for a position.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getDocumentsForPosition(positionId) -@router.get("/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) +@router.get("/{instanceId}/position-documents/document/{documentId}", response_model=List[TrusteePositionDocument]) @limiter.limit("30/minute") async def getPositionsForDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), documentId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> List[TrusteePositionDocument]: """Get all position links for a document.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) return interface.getPositionsForDocument(documentId) -@router.post("/position-documents", response_model=TrusteePositionDocument, status_code=201) +@router.post("/{instanceId}/position-documents", response_model=TrusteePositionDocument, status_code=201) @limiter.limit("10/minute") async def createPositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), data: TrusteePositionDocument = Body(...), context: RequestContext = Depends(getRequestContext) ) -> TrusteePositionDocument: """Create a new position-document link.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.createPositionDocument(data.model_dump()) if not result: raise HTTPException(status_code=400, detail="Failed to create link") return result -@router.delete("/position-documents/{linkId}") +@router.delete("/{instanceId}/position-documents/{linkId}") @limiter.limit("10/minute") async def deletePositionDocument( request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), linkId: str = Path(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: """Delete a position-document link.""" - interface = getInterface(context.user, mandateId=context.mandateId) + mandateId = await _validateInstanceAccess(instanceId, context) + + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) existing = interface.getPositionDocument(linkId) if not existing: raise HTTPException(status_code=404, detail=f"Link {linkId} not found") diff --git a/modules/routes/routeFeatureAutomation.py b/modules/routes/routeFeatureWorkflow.py similarity index 95% rename from modules/routes/routeFeatureAutomation.py rename to modules/routes/routeFeatureWorkflow.py index ea58e271..ac002332 100644 --- a/modules/routes/routeFeatureAutomation.py +++ b/modules/routes/routeFeatureWorkflow.py @@ -11,7 +11,7 @@ from fastapi import status import logging # Import interfaces and models -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.auth import limiter, getRequestContext, requireSysAdmin, RequestContext # Configure logger @@ -75,7 +75,7 @@ async def sync_all_automation_events( This will register/remove events based on active flags. """ try: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface + from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.features.workflow import syncAutomationEvents @@ -126,7 +126,7 @@ async def remove_event( # Update automation's eventId if it exists if eventId.startswith("automation."): automation_id = eventId.replace("automation.", "") - chatInterface = interfaceDbChatObjects.getInterface(context.user) + chatInterface = interfaceDbChatbot.getInterface(context.user) automation = chatInterface.getAutomationDefinition(automation_id) if automation and getattr(automation, "eventId", None) == eventId: chatInterface.updateAutomationDefinition(automation_id, {"eventId": None}) diff --git a/modules/routes/routeGdpr.py b/modules/routes/routeGdpr.py index 3eef1980..53375849 100644 --- a/modules/routes/routeGdpr.py +++ b/modules/routes/routeGdpr.py @@ -204,12 +204,13 @@ async def exportUserData( for inv in invitationsUsed ] - # Audit log - audit_logger.logSecurityEvent( + # Audit log - GDPR Article 15 data export + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_data_export", - details="User requested data export (Article 15)" + details="User requested data export (GDPR Article 15 - Right of Access)", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} exported personal data (GDPR Art. 15)") @@ -304,12 +305,13 @@ async def exportPortableData( "about": portableData } - # Audit log - audit_logger.logSecurityEvent( + # Audit log - GDPR Article 20 data portability + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_data_portability", - details="User requested portable data export (Article 20)" + details="User requested portable data export (GDPR Article 20 - Right to Data Portability)", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} exported portable data (GDPR Art. 20)") @@ -431,12 +433,13 @@ async def deleteAccount( rootInterface.db.recordDelete(User, str(currentUser.id)) deletedData.append("User account deleted") - # Audit log (before user is deleted) - audit_logger.logSecurityEvent( + # Audit log (before user is deleted) - GDPR Article 17 account deletion + audit_logger.logGdprEvent( userId=str(currentUser.id), mandateId="system", action="gdpr_account_deletion", - details=f"User deleted own account (Article 17). Data: {', '.join(deletedData)}" + details=f"User deleted own account (GDPR Article 17 - Right to Erasure). Data: {', '.join(deletedData)}", + ipAddress=request.client.host if request.client else None ) logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17)") diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 4e61a045..2a8f65fd 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -624,7 +624,10 @@ async def logout( userId=str(currentUser.id), mandateId="system", action="logout", - successInfo="google_auth_logout" + successInfo="google_auth_logout", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 4b64b671..8589db04 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -142,7 +142,10 @@ async def login( userId=str(user.id), mandateId="system", action="login", - successInfo="local_auth_success" + successInfo="local_auth_success", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails @@ -171,10 +174,13 @@ async def login( try: from modules.shared.auditLogger import audit_logger audit_logger.logUserAccess( - userId="unknown", - mandateId="unknown", - action="login", - successInfo=f"failed: {error_msg}" + userId=formData.username or "unknown", + mandateId="system", + action="login_failed", + successInfo=f"failed: {error_msg}", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=False ) except Exception: # Don't fail if audit logging fails @@ -438,7 +444,10 @@ async def logout(request: Request, response: Response, currentUser: User = Depen userId=str(currentUser.id), mandateId="system", action="logout", - successInfo=f"revoked_tokens: {revoked}" + successInfo=f"revoked_tokens: {revoked}", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index c145d1d3..77dc9885 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -634,7 +634,10 @@ async def logout( userId=str(currentUser.id), mandateId="system", action="logout", - successInfo="microsoft_auth_logout" + successInfo="microsoft_auth_logout", + ipAddress=request.client.host if request.client else None, + userAgent=request.headers.get("user-agent"), + success=True ) except Exception: # Don't fail if audit logging fails diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 6d2f78ee..1c7d9e80 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -14,12 +14,12 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Respon from modules.auth import limiter, getCurrentUser # Import interfaces -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects -from modules.interfaces.interfaceDbChatObjects import getInterface +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot +from modules.interfaces.interfaceDbChatbot import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( ChatWorkflow, ChatMessage, ChatLog, @@ -45,7 +45,7 @@ router = APIRouter( ) def getServiceChat(currentUser: User): - return interfaceDbChatObjects.getInterface(currentUser) + return interfaceDbChatbot.getInterface(currentUser) # Consolidated endpoint for getting all workflows @router.get("/", response_model=PaginatedResponse[ChatWorkflow]) diff --git a/modules/services/__init__.py b/modules/services/__init__.py index 7033bfbb..b75b2454 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -3,7 +3,7 @@ from typing import Any, Optional from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow class PublicService: """Lightweight proxy exposing only public callable attributes of a target. @@ -49,7 +49,7 @@ class Services: # Initialize interfaces with explicit mandateId - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface + from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface self.interfaceDbChat = getChatInterface(user, mandateId=mandateId) from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface @@ -58,7 +58,7 @@ class Services: from modules.interfaces.interfaceDbComponentObjects import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) - from modules.interfaces.interfaceDbTrusteeObjects import getInterface as getTrusteeInterface + from modules.interfaces.interfaceDbTrustee import getInterface as getTrusteeInterface self.interfaceDbTrustee = getTrusteeInterface(user, mandateId=mandateId) # Expose RBAC directly on services for convenience diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index cd86c6a8..55bd2544 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -6,7 +6,7 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument +from modules.datamodels.datamodelChatbot import PromptPlaceholder, ChatDocument from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py index a866f68f..ec6a26d2 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py index 821851a4..e90ecfeb 100644 --- a/modules/services/serviceAi/subDocumentIntents.py +++ b/modules/services/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 137dcd05..0c82929d 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChatbot import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 13739dea..64678f54 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -11,7 +11,7 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index a49b78c7..adc0ea78 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -6,7 +6,7 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelChatbot import ChatDocument from modules.services.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index d3d4dfda..5d3a8497 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -157,11 +157,11 @@ class UtilsService: def storeDebugMessageAndDocuments(self, message, currentUser): """ - Wrapper to store debug messages and documents via interfaceDbChatObjects. - Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatObjects. + Wrapper to store debug messages and documents via interfaceDbChatbot. + Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatbot. """ try: - from modules.interfaces.interfaceDbChatObjects import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChatbot import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/shared/auditLogger.py b/modules/shared/auditLogger.py index 46e80d5f..48cd8fdb 100644 --- a/modules/shared/auditLogger.py +++ b/modules/shared/auditLogger.py @@ -4,201 +4,471 @@ Audit Logging System for PowerOn Gateway This module provides centralized audit logging functionality for security events, -user actions, and system access patterns. +user actions, and system access patterns. Logs are stored in the database for +GDPR compliance and security monitoring. + +GDPR Requirements Addressed: +- Article 5(1)(f): Integrity and confidentiality - secure audit trail +- Article 17: Right to erasure - audit log retention with automatic cleanup +- Article 30: Records of processing activities - comprehensive event logging """ import logging -import os from datetime import datetime from typing import Optional, Dict, Any -from logging.handlers import RotatingFileHandler + from modules.shared.configuration import APP_CONFIG +from modules.shared.timeUtils import getUtcTimestamp - -class DailyRotatingFileHandler(RotatingFileHandler): - """ - A rotating file handler that automatically switches to a new file when the date changes. - The log file name includes the current date and switches at midnight. - """ - - def __init__(self, logDir, filenamePrefix, maxBytes=10485760, backupCount=5, **kwargs): - self.logDir = logDir - self.filenamePrefix = filenamePrefix - self.currentDate = None - self.currentFile = None - - # Initialize with today's file - self._updateFileIfNeeded() - - # Call parent constructor with current file - super().__init__(self.currentFile, maxBytes=maxBytes, backupCount=backupCount, **kwargs) - - def _updateFileIfNeeded(self): - """Update the log file if the date has changed""" - today = datetime.now().strftime("%Y%m%d") - - if self.currentDate != today: - self.currentDate = today - newFile = os.path.join(self.logDir, f"{self.filenamePrefix}_{today}.log") - - if self.currentFile != newFile: - self.currentFile = newFile - return True - return False - - def emit(self, record): - """Emit a log record, switching files if date has changed""" - # Check if we need to switch to a new file - if self._updateFileIfNeeded(): - # Close current file and open new one - if self.stream: - self.stream.close() - self.stream = None - - # Update the baseFilename for the parent class - self.baseFilename = self.currentFile - # Reopen the stream - if not self.delay: - self.stream = self._open() - - # Call parent emit method - super().emit(record) +logger = logging.getLogger(__name__) class AuditLogger: - """Centralized audit logging system""" + """ + Centralized audit logging system with database storage. + + Logs security-relevant events to PostgreSQL for: + - GDPR compliance + - Security monitoring + - Access tracking + - Incident investigation + """ def __init__(self): - self.logger = None - self._setupAuditLogger() - - def _setupAuditLogger(self): - """Setup the audit logger with daily file rotation""" + self._db = None + self._modelClass = None + self._initialized = False + self._fallbackToStdout = True + + def _ensureInitialized(self) -> bool: + """Lazily initialize database connection to avoid circular imports.""" + if self._initialized: + return self._db is not None + + self._initialized = True + try: - # Get log directory from config - logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") - if not os.path.isabs(logDir): - # If relative path, make it relative to the gateway directory - gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - logDir = os.path.join(gatewayDir, logDir) + from modules.datamodels.datamodelAudit import AuditLogEntry + from modules.connectors.connectorDbPostgre import DatabaseConnector - # Ensure log directory exists - os.makedirs(logDir, exist_ok=True) + self._modelClass = AuditLogEntry - # Create audit logger - self.logger = logging.getLogger('audit') - self.logger.setLevel(logging.INFO) + # Get database configuration + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_app" # Store audit logs in the main app database + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) - # Remove any existing handlers to avoid duplicates - for handler in self.logger.handlers[:]: - self.logger.removeHandler(handler) - - # Create daily rotating file handler for audit log - rotationSize = int(APP_CONFIG.get("APP_LOGGING_ROTATION_SIZE", 10485760)) # Default: 10MB - backupCount = int(APP_CONFIG.get("APP_LOGGING_BACKUP_COUNT", 5)) - - fileHandler = DailyRotatingFileHandler( - logDir=logDir, - filenamePrefix="log_audit", - maxBytes=rotationSize, - backupCount=backupCount + # Create database connector with system user context + self._db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId="system" # Audit logs are created by system ) - # Create formatter for audit log - auditFormatter = logging.Formatter( - fmt="%(asctime)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" - ) - fileHandler.setFormatter(auditFormatter) + # Initialize database and ensure table exists + self._db.initDbSystem() + self._db._ensureTableExists(AuditLogEntry) - # Add handler to logger - self.logger.addHandler(fileHandler) - - # Prevent propagation to root logger - self.logger.propagate = False + logger.info("AuditLogger database connection initialized successfully") + return True except Exception as e: - # Fallback to standard logger if audit setup fails - self.logger = logging.getLogger(__name__) - self.logger.error(f"Failed to setup audit logger: {str(e)}") + logger.warning(f"AuditLogger database initialization failed, using fallback logging: {e}") + self._db = None + return False - def logEvent(self, - userId: str, - mandateId: str, - category: str, - action: str, - details: str = "", - timestamp: Optional[datetime] = None) -> None: + def _logToFallback(self, entry: Dict[str, Any]) -> None: + """Log to standard logger as fallback when database is unavailable.""" + if self._fallbackToStdout: + fallbackMsg = ( + f"AUDIT | {entry.get('timestamp', '')} | " + f"{entry.get('userId', '')} | {entry.get('mandateId', '')} | " + f"{entry.get('category', '')} | {entry.get('action', '')} | " + f"{entry.get('details', '')}" + ) + logging.getLogger('audit.fallback').info(fallbackMsg) + + def logEvent( + self, + userId: str, + mandateId: Optional[str] = None, + category: str = "system", + action: str = "", + details: str = "", + featureInstanceId: Optional[str] = None, + resourceType: Optional[str] = None, + resourceId: Optional[str] = None, + ipAddress: Optional[str] = None, + userAgent: Optional[str] = None, + success: bool = True, + errorMessage: Optional[str] = None, + username: Optional[str] = None, + timestamp: Optional[float] = None + ) -> Optional[str]: """ - Log an audit event + Log an audit event to the database. Args: - userId: User identifier - mandateId: Mandate identifier (can be empty if not applicable) - category: Event category (e.g., 'key', 'access', 'data') - action: Specific action (e.g., 'decode', 'login', 'logout') + userId: User identifier (or 'system' for system events) + mandateId: Mandate context (can be None for system-level events) + category: Event category (access, key, data, security, gdpr, permission, system) + action: Specific action performed details: Additional details about the event + featureInstanceId: Feature instance context (if applicable) + resourceType: Type of resource affected + resourceId: ID of the affected resource + ipAddress: Client IP address + userAgent: Client user agent + success: Whether the action was successful + errorMessage: Error message if action failed + username: Username at the time of event (for historical reference) timestamp: Optional custom timestamp (defaults to current time) + + Returns: + ID of the created audit log entry, or None if logging failed """ try: - if not self.logger: - return - - # Use provided timestamp or current time - if timestamp is None: - timestamp = datetime.now() - - # Format the audit log entry - # Format: timestamp | userid | mandateid | category | action | details - auditEntry = f"{userId} | {mandateId} | {category} | {action} | {details}" - - # Log the event - self.logger.info(auditEntry) + # Prepare the entry data + entryData = { + "timestamp": timestamp if timestamp else getUtcTimestamp(), + "userId": userId or "unknown", + "username": username, + "mandateId": mandateId, + "featureInstanceId": featureInstanceId, + "category": category, + "action": action, + "resourceType": resourceType, + "resourceId": resourceId, + "details": details if details else None, + "ipAddress": ipAddress, + "userAgent": userAgent, + "success": success, + "errorMessage": errorMessage + } + # Try to write to database + if self._ensureInitialized() and self._db: + from modules.datamodels.datamodelAudit import AuditLogEntry + + entry = AuditLogEntry(**entryData) + created = self._db.recordCreate(AuditLogEntry, entry.model_dump()) + + if created and created.get("id"): + return created["id"] + else: + self._logToFallback(entryData) + return None + else: + # Use fallback logging + self._logToFallback(entryData) + return None + except Exception as e: - # Use standard logger as fallback - logging.getLogger(__name__).error(f"Failed to log audit event: {str(e)}") + logger.error(f"Failed to log audit event: {e}") + # Try fallback + try: + self._logToFallback(entryData) + except Exception: + pass + return None - def logKeyAccess(self, userId: str, mandateId: str, keyName: str, action: str) -> None: - """Log key access events (decode/encode)""" - self.logEvent( + # ===== Convenience Methods for Common Event Types ===== + + def logKeyAccess( + self, + userId: str, + mandateId: str, + keyName: str, + action: str, + ipAddress: Optional[str] = None + ) -> Optional[str]: + """Log key access events (encode/decode).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="key", action=action, - details=keyName + details=f"Key: {keyName}", + resourceType="EncryptionKey", + resourceId=keyName, + ipAddress=ipAddress ) - def logUserAccess(self, userId: str, mandateId: str, action: str, successInfo: str = "") -> None: - """Log user access events (login/logout)""" - self.logEvent( + def logUserAccess( + self, + userId: str, + mandateId: str, + action: str, + successInfo: str = "", + ipAddress: Optional[str] = None, + userAgent: Optional[str] = None, + success: bool = True + ) -> Optional[str]: + """Log user access events (login/logout).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="access", action=action, - details=successInfo + details=successInfo, + ipAddress=ipAddress, + userAgent=userAgent, + success=success ) - def logDataAccess(self, userId: str, mandateId: str, action: str, details: str = "") -> None: - """Log data access events""" - self.logEvent( + def logDataAccess( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + resourceType: Optional[str] = None, + resourceId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ) -> Optional[str]: + """Log data access events (CRUD operations).""" + return self.logEvent( userId=userId, mandateId=mandateId, category="data", action=action, - details=details + details=details, + resourceType=resourceType, + resourceId=resourceId, + featureInstanceId=featureInstanceId ) - def logSecurityEvent(self, userId: str, mandateId: str, action: str, details: str = "") -> None: - """Log security-related events""" - self.logEvent( + def logSecurityEvent( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + ipAddress: Optional[str] = None, + success: bool = True, + errorMessage: Optional[str] = None + ) -> Optional[str]: + """Log security-related events.""" + return self.logEvent( userId=userId, mandateId=mandateId, category="security", action=action, - details=details + details=details, + ipAddress=ipAddress, + success=success, + errorMessage=errorMessage ) + + def logGdprEvent( + self, + userId: str, + mandateId: str, + action: str, + details: str = "", + ipAddress: Optional[str] = None + ) -> Optional[str]: + """Log GDPR-specific events (data export, deletion, etc.).""" + return self.logEvent( + userId=userId, + mandateId=mandateId, + category="gdpr", + action=action, + details=details, + ipAddress=ipAddress + ) + + def logPermissionChange( + self, + userId: str, + mandateId: str, + action: str, + targetUserId: str, + details: str = "", + resourceType: Optional[str] = None, + resourceId: Optional[str] = None + ) -> Optional[str]: + """Log permission/role changes.""" + return self.logEvent( + userId=userId, + mandateId=mandateId, + category="permission", + action=action, + details=f"Target user: {targetUserId}. {details}", + resourceType=resourceType, + resourceId=resourceId + ) + + # ===== Audit Log Query Methods ===== + + def getAuditLogs( + self, + userId: Optional[str] = None, + mandateId: Optional[str] = None, + category: Optional[str] = None, + action: Optional[str] = None, + fromTimestamp: Optional[float] = None, + toTimestamp: Optional[float] = None, + limit: int = 100 + ) -> list: + """ + Query audit logs from database. + + Args: + userId: Filter by user ID + mandateId: Filter by mandate ID + category: Filter by category + action: Filter by action + fromTimestamp: Filter events after this timestamp + toTimestamp: Filter events before this timestamp + limit: Maximum number of records to return + + Returns: + List of audit log entries + """ + if not self._ensureInitialized() or not self._db: + return [] + + try: + from modules.datamodels.datamodelAudit import AuditLogEntry + + # Build filter + recordFilter = {} + if userId: + recordFilter["userId"] = userId + if mandateId: + recordFilter["mandateId"] = mandateId + if category: + recordFilter["category"] = category + if action: + recordFilter["action"] = action + + # Query database + records = self._db.getRecordset( + AuditLogEntry, + recordFilter=recordFilter if recordFilter else None, + orderBy="timestamp DESC" + ) + + # Apply timestamp filtering in Python (PostgreSQL connector may not support $gt/$lt) + if fromTimestamp or toTimestamp: + filteredRecords = [] + for record in records: + ts = record.get("timestamp", 0) + if fromTimestamp and ts < fromTimestamp: + continue + if toTimestamp and ts > toTimestamp: + continue + filteredRecords.append(record) + records = filteredRecords + + # Apply limit + return records[:limit] + + except Exception as e: + logger.error(f"Failed to query audit logs: {e}") + return [] + + # ===== Cleanup Methods ===== + + def cleanupOldEntries(self, retentionDays: int = 365) -> int: + """ + Remove audit log entries older than the retention period. + + GDPR Note: Audit logs should be retained for a reasonable period + for security and compliance purposes, but not indefinitely. + Default retention is 1 year (365 days). + + Args: + retentionDays: Number of days to retain audit logs + + Returns: + Number of entries deleted + """ + if not self._ensureInitialized() or not self._db: + logger.warning("Cannot cleanup audit logs: database not initialized") + return 0 + + try: + from modules.datamodels.datamodelAudit import AuditLogEntry + import time + + # Calculate cutoff timestamp + cutoffTimestamp = time.time() - (retentionDays * 24 * 60 * 60) + + # Query old entries + allRecords = self._db.getRecordset(AuditLogEntry) + oldRecords = [r for r in allRecords if r.get("timestamp", 0) < cutoffTimestamp] + + # Delete old entries + deletedCount = 0 + for record in oldRecords: + recordId = record.get("id") + if recordId: + if self._db.recordDelete(AuditLogEntry, recordId): + deletedCount += 1 + + logger.info(f"Audit log cleanup: removed {deletedCount} entries older than {retentionDays} days") + + # Log the cleanup action itself + self.logEvent( + userId="system", + mandateId="system", + category="system", + action="audit_cleanup", + details=f"Removed {deletedCount} entries older than {retentionDays} days" + ) + + return deletedCount + + except Exception as e: + logger.error(f"Failed to cleanup audit logs: {e}") + return 0 # Global audit logger instance audit_logger = AuditLogger() + + +# ===== Scheduler Integration ===== + +async def runAuditLogCleanup() -> None: + """ + Scheduled task to cleanup old audit log entries. + Called by the event scheduler. + """ + try: + retentionDays = int(APP_CONFIG.get("AUDIT_LOG_RETENTION_DAYS", 365)) + deletedCount = audit_logger.cleanupOldEntries(retentionDays=retentionDays) + logger.info(f"Scheduled audit log cleanup completed: {deletedCount} entries removed") + except Exception as e: + logger.error(f"Scheduled audit log cleanup failed: {e}") + + +def registerAuditLogCleanupScheduler() -> None: + """ + Register the audit log cleanup job with the event scheduler. + Should be called during application startup. + """ + try: + from modules.shared.eventManagement import eventManager + + # Run cleanup daily at 3 AM + eventManager.registerCron( + jobId="audit_log_cleanup", + func=runAuditLogCleanup, + cronKwargs={ + "hour": "3", + "minute": "0" + } + ) + + logger.info("Audit log cleanup scheduler registered (daily at 03:00)") + + except Exception as e: + logger.error(f"Failed to register audit log cleanup scheduler: {e}") diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index 39d6e16f..a3f45261 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 4f9bbd21..77cb361f 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 65e95a32..6c509a9e 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index f804c0b9..bddeb252 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index e32c1965..806679df 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index bb6f8437..96f2609c 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult +from modules.datamodels.datamodelChatbot import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 62b43bce..4c5d8314 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index ff7e896f..c6e6d560 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index 5b90ce13..cccad45d 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index 9991285b..fedbc46b 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 8e3b7185..0b646251 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index 2f011a25..015eb1e3 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index 45b60cad..f00192f6 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index cbec7960..1e44b1cb 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index 631795b3..afa0c5fc 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 55d99654..6e6106b0 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index b997889e..d72e3d55 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 2bd7ab74..49ddece2 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index bbdc2cc7..24c097e8 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index 5ac4e548..adfe13ea 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 59604896..06f26e89 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 2d325d9f..4ff700ca 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index f8831d59..4531859f 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index 9b7fb011..da9f8cd4 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index a4bf18b6..05997512 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index f149e482..287612ff 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index c64a6637..e6c2a276 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 722dbc99..4eac8544 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index 62b6dd94..a9b837aa 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index 318271c3..d0838633 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index 73cdb730..eaf3254f 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index e9361853..ddce6206 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 1f469b80..85d7b123 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 0e4d6ee4..7bde4da7 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ActionResult, ActionItem, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index a4ae05e9..0daac228 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 0fac427c..4b1fcad5 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.datamodels.datamodelChat import WorkflowModeEnum + from modules.datamodels.datamodelChatbot import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index e3131939..85f1f824 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index 770c868a..e8837d65 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskResult, ActionItem +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index f7754eab..b821511b 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.datamodels.datamodelChat import ReviewContext + from modules.datamodels.datamodelChatbot import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.datamodels.datamodelChat import ReviewResult + from modules.datamodels.datamodelChatbot import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 1cdf0d53..b0186be9 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation +from modules.datamodels.datamodelChatbot import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 0be4e029..30e9af4d 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.datamodels.datamodelChatbot import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChat import Observation + from modules.datamodels.datamodelChatbot import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict @@ -452,10 +452,10 @@ def extractLatestRefinementFeedback(context: Any) -> str: # First check for ERROR level logs in workflow if hasattr(context, 'workflow') and context.workflow: try: - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceDbAppObjects import getRootInterface rootInterface = getRootInterface() - interfaceDbChat = interfaceDbChatObjects.getInterface(rootInterface.currentUser) + interfaceDbChat = interfaceDbChatbot.getInterface(rootInterface.currentUser) # Get workflow logs chatData = interfaceDbChat.getUnifiedChatData(context.workflow.id, None) diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 31878033..10932529 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index 11a54ca1..e1d767c4 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 9c9d6c84..317a6cb7 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -6,9 +6,9 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING -from modules.datamodels import datamodelChat -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum +from modules.datamodels import datamodelChatbot +from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -102,7 +102,7 @@ class WorkflowProcessor: self.services.chat.progressLogFinish(operationId, False) raise - async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChat.TaskResult: + async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChatbot.TaskResult: """Execute a task step using the appropriate mode""" import time @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.datamodels.datamodelChat import ActionDocument + from modules.datamodels.datamodelChatbot import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.datamodels.datamodelChat import ChatMessage, ChatDocument, ActionDocument + from modules.datamodels.datamodelChatbot import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index a9b656eb..e6ecdbbd 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.datamodels.datamodelChat import ( +from modules.datamodels.datamodelChatbot import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.datamodels.datamodelChat import TaskContext +from modules.datamodels.datamodelChatbot import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.datamodels.datamodelChat import ActionResult + from modules.datamodels.datamodelChatbot import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 12a374f8..94eb6158 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -85,7 +85,7 @@ class AIModelsTester: self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index 36a8505a..dd5d68e3 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -94,8 +94,8 @@ class MethodAiOperationsTester: logging.getLogger().setLevel(logging.DEBUG) # Import and initialize services - use the same approach as routeChatPlayground - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) # Import and initialize services from modules.services import getInterface as getServices @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.datamodels.datamodelChat import ChatDocument + from modules.datamodels.datamodelChatbot import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.datamodels.datamodelChat import ChatMessage + from modules.datamodels.datamodelChatbot import ChatMessage import time testMessage = ChatMessage( @@ -201,8 +201,8 @@ class MethodAiOperationsTester: # Save message to database if self.services.workflow: - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) messageDict = testMessage.model_dump() interfaceDbChat.createMessage(messageDict) @@ -283,8 +283,8 @@ class MethodAiOperationsTester: maxSteps=5 ) # Save workflow to database - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 51059745..478e9baf 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,10 +42,10 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum import uuid import time - import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects + import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot currentTimestamp = time.time() @@ -67,7 +67,7 @@ class AIBehaviorTester: ) # SAVE workflow to database so it exists for access control - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflowDict = testWorkflow.model_dump() interfaceDbChat.createWorkflow(workflowDict) diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 7ed171c0..3a0cf2a3 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,10 +20,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class WorkflowWithDocumentsTester: @@ -192,7 +192,7 @@ class WorkflowWithDocumentsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id) if not currentWorkflow: @@ -225,7 +225,7 @@ class WorkflowWithDocumentsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index c65fa401..784b1756 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,10 +22,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class WorkflowPromptVariationsTester: @@ -115,7 +115,7 @@ class WorkflowPromptVariationsTester: return False # Get current workflow status - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) currentWorkflow = interfaceDbChat.getWorkflow(workflow.id) if not currentWorkflow: @@ -140,7 +140,7 @@ class WorkflowPromptVariationsTester: def _analyzeWorkflowResults(self, workflow: Any) -> Dict[str, Any]: """Analyze workflow results and extract information.""" - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(workflow.id) if not workflow: diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 3e33c996..9c96d95f 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class DocumentGenerationFormatsTester: @@ -251,7 +251,7 @@ class DocumentGenerationFormatsTester: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -296,7 +296,7 @@ class DocumentGenerationFormatsTester: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index 9ce9b367..a0b744f4 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,10 +21,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class DocumentGenerationFormatsTester10: @@ -248,7 +248,7 @@ class DocumentGenerationFormatsTester10: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -293,7 +293,7 @@ class DocumentGenerationFormatsTester10: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index 266b27e5..e1331cd1 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,10 +23,10 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart -import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects +import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot class CodeGenerationFormatsTester11: @@ -190,7 +190,7 @@ class CodeGenerationFormatsTester11: startTime = time.time() lastStatus = None - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) if timeout is None: print("Waiting indefinitely (no timeout)") @@ -235,7 +235,7 @@ class CodeGenerationFormatsTester11: if not self.workflow: return {"error": "No workflow to analyze"} - interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser) + interfaceDbChat = interfaceDbChatbot.getInterface(self.testUser) workflow = interfaceDbChat.getWorkflow(self.workflow.id) if not workflow: diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index a2b69576..9409a9e6 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index ae502397..b91aa1e7 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 09f6e92c..25a5af8e 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelChatbot import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel diff --git a/tool_db_export_migration.py b/tool_db_export_migration.py new file mode 100644 index 00000000..09aa2f8f --- /dev/null +++ b/tool_db_export_migration.py @@ -0,0 +1,508 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Datenbank Export-Tool für Migration. + +Dieses Script exportiert alle Daten aus ALLEN PowerOn PostgreSQL-Datenbanken +in eine JSON-Datei, die als Migrationsdatensatz verwendet werden kann. + +Datenbanken: + - poweron_app (User, Mandate, RBAC, Features, etc.) + - poweron_chat (Chat-Konversationen und Nachrichten) + - poweron_management (Workflows, Prompts, Connections, etc.) + - poweron_realestate (Real Estate Daten) + - poweron_trustee (Trustee Daten) + +Verwendung: + python tool_db_export_migration.py [--output ] [--pretty] + +Optionen: + --output, -o Pfad zur Ausgabedatei (Standard: migration_export_.json) + --pretty, -p JSON formatiert ausgeben (für bessere Lesbarkeit) + --exclude Komma-getrennte Liste von Tabellen, die ausgeschlossen werden sollen + --include-meta System-Metadaten (_createdAt, _modifiedAt, etc.) beibehalten + --db Nur bestimmte Datenbank(en) exportieren (komma-getrennt) +""" + +import os +import sys +import json +import argparse +import logging +from datetime import datetime +from typing import Dict, List, Any, Optional +from pathlib import Path + +import psycopg2 +import psycopg2.extras + +# Logging konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +# Alle PowerOn Datenbanken +ALL_DATABASES = [ + "poweron_app", # Haupt-App: User, Mandate, RBAC, Features + "poweron_chat", # Chat-Konversationen + "poweron_management", # Workflows, Prompts, Connections + "poweron_realestate", # Real Estate + "poweron_trustee", # Trustee +] + + +def _loadEnvConfig() -> Dict[str, str]: + """Lädt die Konfiguration direkt aus der .env Datei.""" + config = {} + envPath = Path(__file__).parent / '.env' + + if not envPath.exists(): + logger.warning(f"Environment file not found at {envPath}") + return config + + # Versuche verschiedene Encodings + encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] + + for encoding in encodings: + try: + with open(envPath, 'r', encoding=encoding) as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' in line: + key, value = line.split('=', 1) + config[key.strip()] = value.strip() + # Erfolgreich geladen + return config + except UnicodeDecodeError: + continue + except Exception as e: + logger.error(f"Error loading .env file with {encoding}: {e}") + continue + + logger.error(f"Could not load .env file with any encoding") + return config + + +# Globale Konfiguration laden +_ENV_CONFIG = _loadEnvConfig() + + +def _getConfigValue(key: str, default: str = None) -> str: + """Holt einen Konfigurationswert.""" + return _ENV_CONFIG.get(key, os.environ.get(key, default)) + + +def _databaseExists(dbDatabase: str) -> bool: + """Prüft ob eine Datenbank existiert.""" + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + # Verbinde zur postgres Datenbank um zu prüfen + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database="postgres", + user=dbUser, + password=dbPassword + ) + conn.autocommit = True + + with conn.cursor() as cursor: + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (dbDatabase,) + ) + exists = cursor.fetchone() is not None + + conn.close() + return exists + except Exception as e: + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + return False + + +def _getDbConnection(dbDatabase: str): + """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht - übersprungen") + return None + + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database=dbDatabase, + user=dbUser, + password=dbPassword, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.set_client_encoding('UTF8') + return conn + except Exception as e: + logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + raise + + +def _getTables(conn) -> List[str]: + """Gibt alle Tabellennamen in der Datenbank zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """) + tables = [row["table_name"] for row in cursor.fetchall()] + return tables + + +def _getTableData(conn, tableName: str, includeMeta: bool = False) -> List[Dict[str, Any]]: + """Liest alle Daten aus einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'SELECT * FROM "{tableName}"') + rows = cursor.fetchall() + + records = [] + for row in rows: + record = dict(row) + + # Optional: System-Metadaten entfernen + if not includeMeta: + metaFields = ["_createdAt", "_modifiedAt", "_createdBy", "_modifiedBy"] + for field in metaFields: + record.pop(field, None) + + # Konvertiere JSONB-Felder (sind bereits als Dict/List von psycopg2) + for key, value in record.items(): + if isinstance(value, (int, float)): + record[key] = float(value) if isinstance(value, float) else int(value) + + records.append(record) + + return records + + +def _getTableRowCount(conn, tableName: str) -> int: + """Zählt die Anzahl der Zeilen in einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'SELECT COUNT(*) as count FROM "{tableName}"') + result = cursor.fetchone() + return result["count"] if result else 0 + + +def _exportSingleDatabase( + dbDatabase: str, + excludeTables: List[str], + includeMeta: bool +) -> Optional[Dict[str, Any]]: + """Exportiert eine einzelne Datenbank.""" + conn = _getDbConnection(dbDatabase) + + if conn is None: + return None + + try: + allTables = _getTables(conn) + + # System-Tabellen ausschliessen + systemTables = ["_system"] + tablesToExport = [ + t for t in allTables + if t not in systemTables and t not in excludeTables + ] + + dbExport = { + "tables": {}, + "summary": {}, + "tableCount": len(tablesToExport), + "totalRecords": 0 + } + + for tableName in tablesToExport: + try: + records = _getTableData(conn, tableName, includeMeta) + rowCount = len(records) + dbExport["totalRecords"] += rowCount + + dbExport["tables"][tableName] = records + dbExport["summary"][tableName] = {"recordCount": rowCount} + + if rowCount > 0: + logger.info(f" {tableName}: {rowCount} Datensätze") + + except Exception as e: + logger.error(f" Fehler bei Tabelle {tableName}: {e}") + dbExport["tables"][tableName] = [] + dbExport["summary"][tableName] = {"recordCount": 0, "error": str(e)} + + return dbExport + + finally: + conn.close() + + +def exportDatabase( + outputPath: Optional[str] = None, + prettyPrint: bool = False, + excludeTables: Optional[List[str]] = None, + includeMeta: bool = False, + onlyDatabases: Optional[List[str]] = None +) -> str: + """ + Exportiert alle Datenbanken in eine JSON-Datei. + + Args: + outputPath: Pfad zur Ausgabedatei (optional) + prettyPrint: JSON formatiert ausgeben + excludeTables: Liste von Tabellen, die ausgeschlossen werden sollen + includeMeta: System-Metadaten beibehalten + onlyDatabases: Nur diese Datenbanken exportieren + + Returns: + Pfad zur erstellten Exportdatei + """ + excludeTables = excludeTables or [] + + # Welche Datenbanken exportieren? + databasesToExport = onlyDatabases if onlyDatabases else ALL_DATABASES + + # Standard-Ausgabepfad generieren (im Log-Ordner) + if not outputPath: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + logDir = _getConfigValue("APP_LOGGING_LOG_DIR") + if logDir and os.path.isabs(logDir): + outputDir = logDir + else: + outputDir = os.path.join(os.path.dirname(__file__), "local", "logs") + os.makedirs(outputDir, exist_ok=True) + outputPath = os.path.join(outputDir, f"migration_export_{timestamp}.json") + + logger.info(f"Starte Export von {len(databasesToExport)} Datenbank(en)...") + logger.info(f"Datenbanken: {', '.join(databasesToExport)}") + + # Export-Struktur erstellen + exportData = { + "meta": { + "exportedAt": datetime.utcnow().isoformat() + "Z", + "exportedFrom": _getConfigValue("APP_ENV_LABEL", "unknown"), + "version": "1.0", + "databaseCount": 0, + "totalTables": 0, + "totalRecords": 0, + "excludedTables": excludeTables, + "includesMeta": includeMeta + }, + "databases": {} + } + + # Jede Datenbank exportieren + for dbName in databasesToExport: + logger.info(f"Exportiere Datenbank: {dbName}") + + dbExport = _exportSingleDatabase(dbName, excludeTables, includeMeta) + + if dbExport is not None: + exportData["databases"][dbName] = dbExport + exportData["meta"]["databaseCount"] += 1 + exportData["meta"]["totalTables"] += dbExport["tableCount"] + exportData["meta"]["totalRecords"] += dbExport["totalRecords"] + logger.info(f" -> {dbExport['tableCount']} Tabellen, {dbExport['totalRecords']} Datensätze") + else: + logger.info(f" -> Übersprungen (existiert nicht)") + + # JSON-Datei schreiben + logger.info(f"Schreibe Exportdatei: {outputPath}") + + with open(outputPath, "w", encoding="utf-8") as f: + if prettyPrint: + json.dump(exportData, f, indent=2, ensure_ascii=False, default=str) + else: + json.dump(exportData, f, ensure_ascii=False, default=str) + + # Dateigrösse berechnen + fileSize = os.path.getsize(outputPath) + fileSizeStr = _formatFileSize(fileSize) + + logger.info(f"Export abgeschlossen!") + logger.info(f" Datenbanken: {exportData['meta']['databaseCount']}") + logger.info(f" Tabellen: {exportData['meta']['totalTables']}") + logger.info(f" Datensätze: {exportData['meta']['totalRecords']}") + logger.info(f" Dateigrösse: {fileSizeStr}") + logger.info(f" Ausgabedatei: {outputPath}") + + return outputPath + + +def _formatFileSize(sizeBytes: int) -> str: + """Formatiert Dateigrösse in lesbares Format.""" + for unit in ['B', 'KB', 'MB', 'GB']: + if sizeBytes < 1024: + return f"{sizeBytes:.2f} {unit}" + sizeBytes /= 1024 + return f"{sizeBytes:.2f} TB" + + +def printDatabaseSummary(): + """Zeigt eine Zusammenfassung aller Datenbanken an.""" + print("\n" + "=" * 70) + print("DATENBANK ZUSAMMENFASSUNG - ALLE POWEREON DATENBANKEN") + print("=" * 70) + print(f"Umgebung: {_getConfigValue('APP_ENV_LABEL', 'unknown')}") + print(f"Host: {_getConfigValue('DB_HOST', 'localhost')}") + print("=" * 70) + + grandTotalRecords = 0 + grandTotalTables = 0 + + for dbName in ALL_DATABASES: + print(f"\n{dbName}") + print("-" * 70) + + conn = _getDbConnection(dbName) + if conn is None: + print(" (Datenbank existiert nicht)") + continue + + try: + tables = _getTables(conn) + dbTotalRecords = 0 + + print(f" {'Tabelle':<45} {'Datensätze':>15}") + print(f" {'-' * 45} {'-' * 15}") + + for tableName in tables: + if tableName.startswith("_"): + continue # System-Tabellen überspringen + count = _getTableRowCount(conn, tableName) + dbTotalRecords += count + if count > 0: # Nur nicht-leere Tabellen anzeigen + print(f" {tableName:<45} {count:>15}") + + print(f" {'-' * 45} {'-' * 15}") + print(f" {'Gesamt':<45} {dbTotalRecords:>15}") + + grandTotalRecords += dbTotalRecords + grandTotalTables += len([t for t in tables if not t.startswith("_")]) + + finally: + conn.close() + + print("\n" + "=" * 70) + print(f"GESAMTÜBERSICHT") + print(f" Datenbanken: {len(ALL_DATABASES)}") + print(f" Tabellen: {grandTotalTables}") + print(f" Datensätze: {grandTotalRecords}") + print("=" * 70 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Exportiert alle PowerOn Datenbank-Daten für Migration", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Datenbanken: + poweron_app - User, Mandate, RBAC, Features + poweron_chat - Chat-Konversationen + poweron_management - Workflows, Prompts, Connections + poweron_realestate - Real Estate Daten + poweron_trustee - Trustee Daten + +Beispiele: + python tool_db_export_migration.py + python tool_db_export_migration.py --pretty + python tool_db_export_migration.py -o backup.json --pretty + python tool_db_export_migration.py --db poweron_app,poweron_chat + python tool_db_export_migration.py --exclude Token,AuthEvent --include-meta + python tool_db_export_migration.py --summary + """ + ) + + parser.add_argument( + "-o", "--output", + help="Pfad zur Ausgabedatei", + type=str, + default=None + ) + + parser.add_argument( + "-p", "--pretty", + help="JSON formatiert ausgeben", + action="store_true" + ) + + parser.add_argument( + "--exclude", + help="Komma-getrennte Liste von Tabellen zum Ausschliessen", + type=str, + default="" + ) + + parser.add_argument( + "--include-meta", + help="System-Metadaten (_createdAt, etc.) beibehalten", + action="store_true" + ) + + parser.add_argument( + "--db", + help="Nur bestimmte Datenbank(en) exportieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--summary", + help="Nur Zusammenfassung anzeigen (kein Export)", + action="store_true" + ) + + args = parser.parse_args() + + # Nur Zusammenfassung anzeigen + if args.summary: + printDatabaseSummary() + return + + # Exclude-Liste parsen + excludeTables = [] + if args.exclude: + excludeTables = [t.strip() for t in args.exclude.split(",") if t.strip()] + + # Datenbank-Liste parsen + onlyDatabases = None + if args.db: + onlyDatabases = [db.strip() for db in args.db.split(",") if db.strip()] + + # Export durchführen + try: + outputPath = exportDatabase( + outputPath=args.output, + prettyPrint=args.pretty, + excludeTables=excludeTables, + includeMeta=args.include_meta, + onlyDatabases=onlyDatabases + ) + print(f"\n Export erfolgreich: {outputPath}\n") + + except Exception as e: + logger.error(f"Export fehlgeschlagen: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tool_db_import_migration.py b/tool_db_import_migration.py new file mode 100644 index 00000000..1ab4e4fe --- /dev/null +++ b/tool_db_import_migration.py @@ -0,0 +1,612 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Datenbank Import-Tool für Migration. + +Dieses Script importiert Daten aus einer JSON-Migrationsdatei +in ALLE PowerOn PostgreSQL-Datenbanken. + +ACHTUNG: Dieses Script kann bestehende Daten überschreiben! +Bitte vor dem Import ein Backup erstellen. + +Datenbanken: + - poweron_app (User, Mandate, RBAC, Features, etc.) + - poweron_chat (Chat-Konversationen und Nachrichten) + - poweron_management (Workflows, Prompts, Connections, etc.) + - poweron_realestate (Real Estate Daten) + - poweron_trustee (Trustee Daten) + +Verwendung: + python tool_db_import_migration.py [--dry-run] [--force] + +Optionen: + --dry-run Simuliert den Import ohne Änderungen + --force Bestätigung überspringen + --clear-first Tabellen vor dem Import leeren + --only-tables Komma-getrennte Liste von Tabellen (nur diese importieren) + --only-db Komma-getrennte Liste von Datenbanken (nur diese importieren) +""" + +import os +import sys +import json +import argparse +import logging +import time +from datetime import datetime +from typing import Dict, List, Any, Optional +from pathlib import Path + +import psycopg2 +import psycopg2.extras + +# Logging konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + +# Alle PowerOn Datenbanken +ALL_DATABASES = [ + "poweron_app", + "poweron_chat", + "poweron_management", + "poweron_realestate", + "poweron_trustee", +] + + +def _loadEnvConfig() -> Dict[str, str]: + """Lädt die Konfiguration direkt aus der .env Datei.""" + config = {} + envPath = Path(__file__).parent / '.env' + + if not envPath.exists(): + logger.warning(f"Environment file not found at {envPath}") + return config + + # Versuche verschiedene Encodings + encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] + + for encoding in encodings: + try: + with open(envPath, 'r', encoding=encoding) as f: + for line in f: + line = line.strip() + if not line or line.startswith('#'): + continue + if '=' in line: + key, value = line.split('=', 1) + config[key.strip()] = value.strip() + # Erfolgreich geladen + return config + except UnicodeDecodeError: + continue + except Exception as e: + logger.error(f"Error loading .env file with {encoding}: {e}") + continue + + logger.error(f"Could not load .env file with any encoding") + return config + + +# Globale Konfiguration laden +_ENV_CONFIG = _loadEnvConfig() + + +def _getConfigValue(key: str, default: str = None) -> str: + """Holt einen Konfigurationswert.""" + return _ENV_CONFIG.get(key, os.environ.get(key, default)) + + +def _getUtcTimestamp() -> float: + """Gibt den aktuellen UTC-Timestamp zurück.""" + return time.time() + + +def _databaseExists(dbDatabase: str) -> bool: + """Prüft ob eine Datenbank existiert.""" + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database="postgres", + user=dbUser, + password=dbPassword + ) + conn.autocommit = True + + with conn.cursor() as cursor: + cursor.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (dbDatabase,) + ) + exists = cursor.fetchone() is not None + + conn.close() + return exists + except Exception as e: + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + return False + + +def _getDbConnection(dbDatabase: str, autocommit: bool = False): + """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht") + return None + + dbHost = _getConfigValue("DB_HOST", "localhost") + dbUser = _getConfigValue("DB_USER") + dbPassword = _getConfigValue("DB_PASSWORD_SECRET") + dbPort = int(_getConfigValue("DB_PORT", "5432")) + + try: + conn = psycopg2.connect( + host=dbHost, + port=dbPort, + database=dbDatabase, + user=dbUser, + password=dbPassword, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.set_client_encoding('UTF8') + conn.autocommit = autocommit + return conn + except Exception as e: + logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + raise + + +def _getExistingTables(conn) -> List[str]: + """Gibt alle Tabellennamen in der Datenbank zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """) + tables = [row["table_name"] for row in cursor.fetchall()] + return tables + + +def _getTableColumns(conn, tableName: str) -> List[str]: + """Gibt alle Spalten einer Tabelle zurück.""" + with conn.cursor() as cursor: + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = %s AND table_schema = 'public' + """, (tableName,)) + columns = [row["column_name"] for row in cursor.fetchall()] + return columns + + +def _clearTable(conn, tableName: str): + """Löscht alle Daten aus einer Tabelle.""" + with conn.cursor() as cursor: + cursor.execute(f'DELETE FROM "{tableName}"') + + +def _insertRecord(conn, tableName: str, record: Dict[str, Any], existingColumns: List[str]) -> bool: + """Fügt einen Datensatz in eine Tabelle ein (UPSERT).""" + filteredRecord = {k: v for k, v in record.items() if k in existingColumns} + + if not filteredRecord: + return False + + # Metadaten hinzufügen falls nicht vorhanden + currentTime = _getUtcTimestamp() + if "_createdAt" not in filteredRecord and "_createdAt" in existingColumns: + filteredRecord["_createdAt"] = currentTime + if "_modifiedAt" in existingColumns: + filteredRecord["_modifiedAt"] = currentTime + + columns = list(filteredRecord.keys()) + values = [] + + for col in columns: + value = filteredRecord[col] + if isinstance(value, (dict, list)): + values.append(json.dumps(value)) + else: + values.append(value) + + colNames = ", ".join([f'"{col}"' for col in columns]) + placeholders = ", ".join(["%s"] * len(columns)) + + updateCols = [col for col in columns if col not in ["id", "_createdAt", "_createdBy"]] + updateClause = ", ".join([f'"{col}" = EXCLUDED."{col}"' for col in updateCols]) + + if updateClause: + sql = f''' + INSERT INTO "{tableName}" ({colNames}) + VALUES ({placeholders}) + ON CONFLICT ("id") DO UPDATE SET {updateClause} + ''' + else: + sql = f''' + INSERT INTO "{tableName}" ({colNames}) + VALUES ({placeholders}) + ON CONFLICT ("id") DO NOTHING + ''' + + try: + with conn.cursor() as cursor: + cursor.execute(sql, values) + return True + except Exception as e: + logger.error(f"Fehler beim Einfügen in {tableName}: {e}") + return False + + +def loadMigrationFile(filePath: str) -> Dict[str, Any]: + """Lädt die Migrationsdatei.""" + logger.info(f"Lade Migrationsdatei: {filePath}") + + if not os.path.exists(filePath): + raise FileNotFoundError(f"Datei nicht gefunden: {filePath}") + + with open(filePath, "r", encoding="utf-8") as f: + data = json.load(f) + + # Validierung - unterstütze beide Formate (alt: tables, neu: databases) + if "databases" not in data and "tables" not in data: + raise ValueError("Ungültiges Migrationsformat: 'databases' oder 'tables' erforderlich") + + return data + + +def _importSingleDatabase( + dbName: str, + dbData: Dict[str, Any], + dryRun: bool, + clearFirst: bool, + onlyTables: Optional[List[str]] +) -> Dict[str, Any]: + """Importiert Daten in eine einzelne Datenbank.""" + stats = { + "imported": {}, + "skipped": {}, + "errors": {}, + "totalImported": 0, + "totalSkipped": 0, + "totalErrors": 0 + } + + conn = _getDbConnection(dbName) + if conn is None: + logger.warning(f" Datenbank '{dbName}' existiert nicht - übersprungen") + return stats + + try: + existingTables = _getExistingTables(conn) + tables = dbData.get("tables", {}) + + tablesToImport = list(tables.keys()) + if onlyTables: + tablesToImport = [t for t in tablesToImport if t in onlyTables] + + for tableName in tablesToImport: + records = tables[tableName] + + if tableName not in existingTables: + logger.warning(f" Tabelle '{tableName}' existiert nicht - übersprungen") + stats["skipped"][tableName] = len(records) + stats["totalSkipped"] += len(records) + continue + + if dryRun: + stats["imported"][tableName] = len(records) + stats["totalImported"] += len(records) + continue + + if clearFirst: + _clearTable(conn, tableName) + + existingColumns = _getTableColumns(conn, tableName) + + imported = 0 + errors = 0 + + for record in records: + if _insertRecord(conn, tableName, record, existingColumns): + imported += 1 + else: + errors += 1 + + stats["imported"][tableName] = imported + stats["totalImported"] += imported + + if errors > 0: + stats["errors"][tableName] = errors + stats["totalErrors"] += errors + + if imported > 0: + logger.info(f" {tableName}: {imported} importiert, {errors} Fehler") + + if not dryRun: + conn.commit() + else: + conn.rollback() + + return stats + + except Exception as e: + conn.rollback() + logger.error(f" Import fehlgeschlagen: {e}") + raise + + finally: + conn.close() + + +def importDatabase( + filePath: str, + dryRun: bool = False, + clearFirst: bool = False, + onlyTables: Optional[List[str]] = None, + onlyDatabases: Optional[List[str]] = None +) -> Dict[str, Any]: + """ + Importiert Daten aus einer Migrationsdatei. + + Args: + filePath: Pfad zur Migrationsdatei + dryRun: Nur simulieren + clearFirst: Tabellen vor Import leeren + onlyTables: Nur diese Tabellen importieren + onlyDatabases: Nur diese Datenbanken importieren + + Returns: + Import-Statistiken + """ + migrationData = loadMigrationFile(filePath) + meta = migrationData.get("meta", {}) + + logger.info(f"Migrationsdatei geladen:") + logger.info(f" Exportiert am: {meta.get('exportedAt', 'unbekannt')}") + logger.info(f" Quelle: {meta.get('exportedFrom', 'unbekannt')}") + + stats = { + "databases": {}, + "totalImported": 0, + "totalSkipped": 0, + "totalErrors": 0 + } + + # Neues Format (mehrere Datenbanken) + if "databases" in migrationData: + databases = migrationData["databases"] + logger.info(f" Datenbanken: {len(databases)}") + logger.info(f" Tabellen: {meta.get('totalTables', 'unbekannt')}") + logger.info(f" Datensätze: {meta.get('totalRecords', 'unbekannt')}") + + for dbName, dbData in databases.items(): + if onlyDatabases and dbName not in onlyDatabases: + continue + + logger.info(f"Importiere Datenbank: {dbName}") + dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) + + stats["databases"][dbName] = dbStats + stats["totalImported"] += dbStats["totalImported"] + stats["totalSkipped"] += dbStats["totalSkipped"] + stats["totalErrors"] += dbStats["totalErrors"] + + # Altes Format (einzelne Datenbank - poweron_app) + elif "tables" in migrationData: + logger.info(" Format: Legacy (einzelne Datenbank)") + dbName = "poweron_app" + dbData = {"tables": migrationData["tables"]} + + if not onlyDatabases or dbName in onlyDatabases: + logger.info(f"Importiere Datenbank: {dbName}") + dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) + + stats["databases"][dbName] = dbStats + stats["totalImported"] = dbStats["totalImported"] + stats["totalSkipped"] = dbStats["totalSkipped"] + stats["totalErrors"] = dbStats["totalErrors"] + + if dryRun: + logger.info("Dry-Run: Keine Änderungen vorgenommen") + + return stats + + +def printImportPreview(filePath: str): + """Zeigt eine Vorschau der zu importierenden Daten.""" + migrationData = loadMigrationFile(filePath) + meta = migrationData.get("meta", {}) + + print("\n" + "=" * 70) + print("IMPORT VORSCHAU") + print("=" * 70) + print(f"Datei: {filePath}") + print(f"Exportiert am: {meta.get('exportedAt', 'unbekannt')}") + print(f"Quelle: {meta.get('exportedFrom', 'unbekannt')}") + + # Neues Format + if "databases" in migrationData: + databases = migrationData["databases"] + print(f"Datenbanken: {len(databases)}") + print("=" * 70) + + grandTotal = 0 + for dbName, dbData in databases.items(): + tables = dbData.get("tables", {}) + dbTotal = sum(len(records) for records in tables.values()) + grandTotal += dbTotal + + print(f"\n{dbName} ({dbTotal} Datensätze)") + print("-" * 70) + print(f" {'Tabelle':<45} {'Datensätze':>15}") + print(f" {'-' * 45} {'-' * 15}") + + for tableName, records in sorted(tables.items()): + if len(records) > 0: + print(f" {tableName:<45} {len(records):>15}") + + print("\n" + "=" * 70) + print(f"GESAMT: {grandTotal} Datensätze") + + # Altes Format + elif "tables" in migrationData: + tables = migrationData["tables"] + print(f"Format: Legacy (poweron_app)") + print("-" * 70) + print(f"{'Tabelle':<45} {'Datensätze':>15}") + print("-" * 70) + + totalRecords = 0 + for tableName, records in sorted(tables.items()): + count = len(records) + totalRecords += count + if count > 0: + print(f"{tableName:<45} {count:>15}") + + print("-" * 70) + print(f"{'GESAMT':<45} {totalRecords:>15}") + + print("=" * 70 + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Importiert Datenbank-Daten aus einer Migrationsdatei", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Datenbanken: + poweron_app - User, Mandate, RBAC, Features + poweron_chat - Chat-Konversationen + poweron_management - Workflows, Prompts, Connections + poweron_realestate - Real Estate Daten + poweron_trustee - Trustee Daten + +Beispiele: + python tool_db_import_migration.py migration_export.json --dry-run + python tool_db_import_migration.py migration_export.json --preview + python tool_db_import_migration.py migration_export.json --force + python tool_db_import_migration.py migration_export.json --clear-first --force + python tool_db_import_migration.py migration_export.json --only-db poweron_app + python tool_db_import_migration.py migration_export.json --only-tables UserInDB,Mandate + """ + ) + + parser.add_argument( + "import_file", + help="Pfad zur Migrationsdatei (JSON)", + type=str + ) + + parser.add_argument( + "--dry-run", + help="Simuliert den Import ohne Änderungen", + action="store_true" + ) + + parser.add_argument( + "--force", + help="Bestätigung überspringen", + action="store_true" + ) + + parser.add_argument( + "--clear-first", + help="Tabellen vor dem Import leeren", + action="store_true" + ) + + parser.add_argument( + "--only-tables", + help="Nur diese Tabellen importieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--only-db", + help="Nur diese Datenbank(en) importieren (komma-getrennt)", + type=str, + default="" + ) + + parser.add_argument( + "--preview", + help="Nur Vorschau anzeigen (kein Import)", + action="store_true" + ) + + args = parser.parse_args() + + # Nur Vorschau anzeigen + if args.preview: + printImportPreview(args.import_file) + return + + # Listen parsen + onlyTables = None + if args.only_tables: + onlyTables = [t.strip() for t in args.only_tables.split(",") if t.strip()] + + onlyDatabases = None + if args.only_db: + onlyDatabases = [db.strip() for db in args.only_db.split(",") if db.strip()] + + # Bestätigung einholen + if not args.dry_run and not args.force: + printImportPreview(args.import_file) + + if args.clear_first: + print("WARNUNG: --clear-first wird ALLE bestehenden Daten in den Zieltabellen löschen!") + + response = input("\nMöchten Sie den Import starten? [y/N]: ") + if response.lower() not in ["y", "yes", "j", "ja"]: + print("Import abgebrochen.") + return + + # Import durchführen + try: + if args.dry_run: + logger.info("=== DRY-RUN MODUS ===") + + stats = importDatabase( + filePath=args.import_file, + dryRun=args.dry_run, + clearFirst=args.clear_first, + onlyTables=onlyTables, + onlyDatabases=onlyDatabases + ) + + print("\n" + "=" * 70) + print("IMPORT ERGEBNIS") + print("=" * 70) + print(f"Importiert: {stats['totalImported']} Datensätze") + print(f"Übersprungen: {stats['totalSkipped']} Datensätze") + print(f"Fehler: {stats['totalErrors']} Datensätze") + + if args.dry_run: + print("\n(Dry-Run: Keine tatsächlichen Änderungen vorgenommen)") + else: + print("\n Import erfolgreich abgeschlossen!") + + print("=" * 70 + "\n") + + except Exception as e: + logger.error(f"Import fehlgeschlagen: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main()