From 77e14147441d69bcf5a0a3bce88559f328fa8ccf Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 20 Jan 2026 00:55:39 +0100 Subject: [PATCH] module testing --- .../{datamodelChatbot.py => datamodelChat.py} | 0 modules/datamodels/datamodelWorkflow.py | 2 +- .../datamodels/datamodelWorkflowActions.py | 2 +- modules/features/chatbot/mainChatbot.py | 4 +- modules/features/workflow/mainWorkflow.py | 2 +- modules/interfaces/interfaceDbChatbot.py | 2 +- modules/routes/routeAdminRbacRoles.py | 11 +- modules/routes/routeDataAutomation.py | 2 +- modules/routes/routeDataMandates.py | 84 +-- modules/routes/routeDataUsers.py | 70 +- modules/routes/routeFeatureChatDynamic.py | 2 +- modules/routes/routeFeatureChatbot.py | 2 +- modules/routes/routeFeatures.py | 255 ++++++-- modules/routes/routeWorkflows.py | 2 +- modules/services/__init__.py | 2 +- modules/services/serviceAi/mainServiceAi.py | 2 +- .../serviceAi/subContentExtraction.py | 2 +- .../services/serviceAi/subDocumentIntents.py | 2 +- .../services/serviceChat/mainServiceChat.py | 2 +- .../mainServiceExtraction.py | 2 +- .../mainServiceGeneration.py | 2 +- modules/shared/dbMultiTenantOptimizations.py | 110 +++- .../methodAi/actions/convertDocument.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methodAi/actions/summarizeDocument.py | 2 +- .../methodAi/actions/translateDocument.py | 2 +- .../methods/methodAi/actions/webResearch.py | 2 +- .../methodChatbot/actions/queryDatabase.py | 2 +- .../methodContext/actions/extractContent.py | 2 +- .../methodContext/actions/getDocumentIndex.py | 2 +- .../methodContext/actions/neutralizeData.py | 2 +- .../actions/triggerPreprocessingServer.py | 2 +- .../methods/methodJira/actions/connectJira.py | 2 +- .../methodJira/actions/createCsvContent.py | 2 +- .../methodJira/actions/createExcelContent.py | 2 +- .../methodJira/actions/exportTicketsAsJson.py | 2 +- .../actions/importTicketsFromJson.py | 2 +- .../methodJira/actions/mergeTicketData.py | 2 +- .../methodJira/actions/parseCsvContent.py | 2 +- .../methodJira/actions/parseExcelContent.py | 2 +- .../composeAndDraftEmailWithContext.py | 2 +- .../methodOutlook/actions/readEmails.py | 2 +- .../methodOutlook/actions/searchEmails.py | 2 +- .../methodOutlook/actions/sendDraftEmail.py | 2 +- .../actions/analyzeFolderUsage.py | 2 +- .../methodSharepoint/actions/copyFile.py | 2 +- .../actions/downloadFileByPath.py | 2 +- .../actions/findDocumentPath.py | 2 +- .../methodSharepoint/actions/findSiteByUrl.py | 2 +- .../methodSharepoint/actions/listDocuments.py | 2 +- .../methodSharepoint/actions/readDocuments.py | 2 +- .../actions/uploadDocument.py | 2 +- .../methodSharepoint/actions/uploadFile.py | 2 +- .../processing/core/actionExecutor.py | 4 +- .../processing/core/messageCreator.py | 4 +- .../workflows/processing/core/taskPlanner.py | 4 +- .../processing/modes/modeAutomation.py | 4 +- .../workflows/processing/modes/modeBase.py | 4 +- .../workflows/processing/modes/modeDynamic.py | 8 +- .../processing/shared/executionState.py | 2 +- .../processing/shared/placeholderFactory.py | 4 +- .../shared/promptGenerationActionsDynamic.py | 2 +- .../shared/promptGenerationTaskplan.py | 2 +- .../workflows/processing/workflowProcessor.py | 12 +- modules/workflows/workflowManager.py | 10 +- tests/functional/test02_ai_models.py | 2 +- tests/functional/test03_ai_operations.py | 6 +- tests/functional/test04_ai_behavior.py | 2 +- .../test05_workflow_with_documents.py | 2 +- .../test06_workflow_prompt_variations.py | 2 +- .../test09_document_generation_formats.py | 2 +- .../test10_document_generation_formats.py | 2 +- .../test11_code_generation_formats.py | 2 +- .../workflows/test_workflow_execution.py | 2 +- tests/unit/workflows/test_state_management.py | 2 +- .../test_architecture_validation.py | 2 +- tool_db_adapt_to_models.py | 428 ++++++++++++ tool_db_export_migration.py | 188 ++++-- tool_db_import_migration.py | 612 ------------------ 81 files changed, 1022 insertions(+), 922 deletions(-) rename modules/datamodels/{datamodelChatbot.py => datamodelChat.py} (100%) create mode 100644 tool_db_adapt_to_models.py delete mode 100644 tool_db_import_migration.py diff --git a/modules/datamodels/datamodelChatbot.py b/modules/datamodels/datamodelChat.py similarity index 100% rename from modules/datamodels/datamodelChatbot.py rename to modules/datamodels/datamodelChat.py diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py index 19117fce..b884382c 100644 --- a/modules/datamodels/datamodelWorkflow.py +++ b/modules/datamodels/datamodelWorkflow.py @@ -14,7 +14,7 @@ from modules.datamodels.datamodelDocref import DocumentReferenceList # Forward references for circular imports (use string annotations) if TYPE_CHECKING: - from modules.datamodels.datamodelChatbot import ChatDocument, ActionResult + from modules.datamodels.datamodelChat import ChatDocument, ActionResult from modules.datamodels.datamodelExtraction import ExtractionOptions diff --git a/modules/datamodels/datamodelWorkflowActions.py b/modules/datamodels/datamodelWorkflowActions.py index 1ca90d51..8bac1fd5 100644 --- a/modules/datamodels/datamodelWorkflowActions.py +++ b/modules/datamodels/datamodelWorkflowActions.py @@ -4,7 +4,7 @@ from typing import Optional, Any, Union, List, Dict, Callable, Awaitable from pydantic import BaseModel, Field -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult from modules.shared.frontendTypes import FrontendType from modules.shared.attributeUtils import registerModelLabels diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index a5222966..43503339 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -13,7 +13,7 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -335,7 +335,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChatbot import ChatLog + from modules.datamodels.datamodelChat import ChatLog # Convert to dict if it's a Pydantic model if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() diff --git a/modules/features/workflow/mainWorkflow.py b/modules/features/workflow/mainWorkflow.py index ab92510c..70a2e9aa 100644 --- a/modules/features/workflow/mainWorkflow.py +++ b/modules/features/workflow/mainWorkflow.py @@ -12,7 +12,7 @@ import logging import json from typing import Dict, Any, Optional -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp from modules.shared.eventManagement import eventManager diff --git a/modules/interfaces/interfaceDbChatbot.py b/modules/interfaces/interfaceDbChatbot.py index 9ded2dd8..c9f87a55 100644 --- a/modules/interfaces/interfaceDbChatbot.py +++ b/modules/interfaces/interfaceDbChatbot.py @@ -16,7 +16,7 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( ChatDocument, ChatStat, ChatLog, diff --git a/modules/routes/routeAdminRbacRoles.py b/modules/routes/routeAdminRbacRoles.py index caa39859..a9397867 100644 --- a/modules/routes/routeAdminRbacRoles.py +++ b/modules/routes/routeAdminRbacRoles.py @@ -363,7 +363,16 @@ async def listUsersWithRoles( interface = getRootInterface() # Get all users (SysAdmin sees all) - users = interface.getUsers() + # Use db.getRecordset with UserInDB (the actual database model) + from modules.datamodels.datamodelUam import User, UserInDB + allUsersData = interface.db.getRecordset(UserInDB) + # Convert to User objects, filtering out sensitive fields + users = [] + for u in allUsersData: + cleanedUser = {k: v for k, v in u.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + users.append(User(**cleanedUser)) # Filter by mandate if specified (via UserMandate table) if mandateId: diff --git a/modules/routes/routeDataAutomation.py b/modules/routes/routeDataAutomation.py index 071f8673..3a3e3ab4 100644 --- a/modules/routes/routeDataAutomation.py +++ b/modules/routes/routeDataAutomation.py @@ -15,7 +15,7 @@ import json # Import interfaces and models from modules.interfaces.interfaceDbChatbot import getInterface as getChatInterface from modules.auth import getCurrentUser, limiter -from modules.datamodels.datamodelChatbot import AutomationDefinition, ChatWorkflow +from modules.datamodels.datamodelChat import AutomationDefinition, ChatWorkflow from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.features.workflow import executeAutomation diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index bf3fbc73..817ab762 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -321,11 +321,11 @@ async def delete_mandate( # User Management within Mandates (Mandate-Admin) # ============================================================================= -@router.get("/{mandateId}/users", response_model=List[MandateUserInfo]) +@router.get("/{targetMandateId}/users", response_model=List[MandateUserInfo]) @limiter.limit("60/minute") async def listMandateUsers( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), context: RequestContext = Depends(getRequestContext) ) -> List[MandateUserInfo]: """ @@ -334,7 +334,7 @@ async def listMandateUsers( Requires Mandate-Admin role or SysAdmin. """ # Check permission - if not _hasMandateAdminRole(context, mandateId) and not context.isSysAdmin: + if not _hasMandateAdminRole(context, targetMandateId) and not context.isSysAdmin: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -344,17 +344,17 @@ async def listMandateUsers( rootInterface = interfaceDbAppObjects.getRootInterface() # Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # Get all UserMandate entries for this mandate userMandates = rootInterface.db.getRecordset( UserMandate, - recordFilter={"mandateId": mandateId} + recordFilter={"mandateId": targetMandateId} ) result = [] @@ -383,18 +383,18 @@ async def listMandateUsers( except HTTPException: raise except Exception as e: - logger.error(f"Error listing users for mandate {mandateId}: {e}") + logger.error(f"Error listing users for mandate {targetMandateId}: {e}") raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to list users: {str(e)}" ) -@router.post("/{mandateId}/users", response_model=UserMandateResponse) +@router.post("/{targetMandateId}/users", response_model=UserMandateResponse) @limiter.limit("30/minute") async def addUserToMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), data: UserMandateCreate = Body(...), context: RequestContext = Depends(getRequestContext) ) -> UserMandateResponse: @@ -405,7 +405,7 @@ async def addUserToMandate( SysAdmin cannot add themselves (Self-Eskalation Prevention). Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID data: User ID and role IDs to assign """ # 1. SysAdmin Self-Eskalation Prevention @@ -416,7 +416,7 @@ async def addUserToMandate( ) # 2. Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required to add users" @@ -426,11 +426,11 @@ async def addUserToMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # 3. Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # 4. Verify target user exists @@ -442,7 +442,7 @@ async def addUserToMandate( ) # 5. Check if user is already a member - existingMembership = rootInterface.getUserMandate(data.targetUserId, mandateId) + existingMembership = rootInterface.getUserMandate(data.targetUserId, targetMandateId) if existingMembership: raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -459,7 +459,7 @@ async def addUserToMandate( ) role = roleRecords[0] roleMandateId = role.get("mandateId") - if roleMandateId and str(roleMandateId) != str(mandateId): + if roleMandateId and str(roleMandateId) != str(targetMandateId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Role {roleId} belongs to a different mandate" @@ -468,14 +468,14 @@ async def addUserToMandate( # 7. Create UserMandate userMandate = rootInterface.createUserMandate( userId=data.targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=data.roleIds ) # 8. Audit - Log permission change with IP address audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="user_added_to_mandate", targetUserId=data.targetUserId, details=f"Roles assigned: {data.roleIds}", @@ -484,14 +484,14 @@ async def addUserToMandate( ) logger.info( - f"User {context.user.id} added user {data.targetUserId} to mandate {mandateId} " + f"User {context.user.id} added user {data.targetUserId} to mandate {targetMandateId} " f"with roles {data.roleIds}" ) return UserMandateResponse( userMandateId=str(userMandate.id), userId=data.targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=data.roleIds, enabled=True ) @@ -506,11 +506,11 @@ async def addUserToMandate( ) -@router.delete("/{mandateId}/users/{targetUserId}", response_model=Dict[str, str]) +@router.delete("/{targetMandateId}/users/{targetUserId}", response_model=Dict[str, str]) @limiter.limit("30/minute") async def removeUserFromMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user to remove"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, str]: @@ -521,11 +521,11 @@ async def removeUserFromMandate( Cannot remove the last admin from a mandate (orphan prevention). Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID targetUserId: User ID to remove """ # Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -535,15 +535,15 @@ async def removeUserFromMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # Verify mandate exists - mandate = rootInterface.getMandate(mandateId) + mandate = rootInterface.getMandate(targetMandateId) if not mandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"Mandate {mandateId} not found" + detail=f"Mandate {targetMandateId} not found" ) # Get user's membership - membership = rootInterface.getUserMandate(targetUserId, mandateId) + membership = rootInterface.getUserMandate(targetUserId, targetMandateId) if not membership: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -551,26 +551,26 @@ async def removeUserFromMandate( ) # Check if this is the last admin (orphan prevention) - if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + if _isLastMandateAdmin(rootInterface, targetMandateId, targetUserId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot remove the last admin from a mandate. Assign another admin first." ) # Delete UserMandate (CASCADE will delete UserMandateRole entries) - rootInterface.deleteUserMandate(targetUserId, mandateId) + rootInterface.deleteUserMandate(targetUserId, targetMandateId) # Audit - Log permission change audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="user_removed_from_mandate", targetUserId=targetUserId, details="User removed from mandate", resourceType="UserMandate" ) - logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {mandateId}") + logger.info(f"User {context.user.id} removed user {targetUserId} from mandate {targetMandateId}") return {"message": "User removed from mandate", "userId": targetUserId} @@ -584,11 +584,11 @@ async def removeUserFromMandate( ) -@router.put("/{mandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) +@router.put("/{targetMandateId}/users/{targetUserId}/roles", response_model=UserMandateResponse) @limiter.limit("30/minute") async def updateUserRolesInMandate( request: Request, - mandateId: str = Path(..., description="ID of the mandate"), + targetMandateId: str = Path(..., description="ID of the mandate"), targetUserId: str = Path(..., description="ID of the user"), roleIds: List[str] = Body(..., description="New role IDs to assign"), context: RequestContext = Depends(getRequestContext) @@ -600,12 +600,12 @@ async def updateUserRolesInMandate( Requires Mandate-Admin role. Args: - mandateId: Target mandate ID + targetMandateId: Target mandate ID targetUserId: User ID to update roleIds: New set of role IDs """ # Check Mandate-Admin permission - if not _hasMandateAdminRole(context, mandateId): + if not _hasMandateAdminRole(context, targetMandateId): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required" @@ -615,7 +615,7 @@ async def updateUserRolesInMandate( rootInterface = interfaceDbAppObjects.getRootInterface() # Get user's membership - membership = rootInterface.getUserMandate(targetUserId, mandateId) + membership = rootInterface.getUserMandate(targetUserId, targetMandateId) if not membership: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -632,7 +632,7 @@ async def updateUserRolesInMandate( ) role = roleRecords[0] roleMandateId = role.get("mandateId") - if roleMandateId and str(roleMandateId) != str(mandateId): + if roleMandateId and str(roleMandateId) != str(targetMandateId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Role {roleId} belongs to a different mandate" @@ -640,11 +640,11 @@ async def updateUserRolesInMandate( # Check if removing admin role would leave mandate without admins currentRoleIds = rootInterface.getRoleIdsForUserMandate(str(membership.id)) - isCurrentlyAdmin = _hasAdminRoleInList(rootInterface, currentRoleIds, mandateId) - willBeAdmin = _hasAdminRoleInList(rootInterface, roleIds, mandateId) + isCurrentlyAdmin = _hasAdminRoleInList(rootInterface, currentRoleIds, targetMandateId) + willBeAdmin = _hasAdminRoleInList(rootInterface, roleIds, targetMandateId) if isCurrentlyAdmin and not willBeAdmin: - if _isLastMandateAdmin(rootInterface, mandateId, targetUserId): + if _isLastMandateAdmin(rootInterface, targetMandateId, targetUserId): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Cannot remove admin role from the last admin. Assign another admin first." @@ -665,7 +665,7 @@ async def updateUserRolesInMandate( # Audit - Log role assignment change audit_logger.logPermissionChange( userId=str(context.user.id), - mandateId=mandateId, + mandateId=targetMandateId, action="role_assigned", targetUserId=targetUserId, details=f"New roles: {roleIds}", @@ -675,13 +675,13 @@ async def updateUserRolesInMandate( logger.info( f"User {context.user.id} updated roles for user {targetUserId} " - f"in mandate {mandateId} to {roleIds}" + f"in mandate {targetMandateId} to {roleIds}" ) return UserMandateResponse( userMandateId=str(membership.id), userId=targetUserId, - mandateId=mandateId, + mandateId=targetMandateId, roleIds=roleIds, enabled=membership.enabled ) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 226f3606..a9ebbab5 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -71,16 +71,43 @@ async def get_users( # MULTI-TENANT: Use mandateId from context (header) # SysAdmin without mandateId can see all users if context.mandateId: - # Get users for specific mandate via UserMandate table - from modules.datamodels.datamodelMembership import UserMandate - userMandates = appInterface.db.getRecordset(UserMandate, recordFilter={"mandateId": str(context.mandateId)}) - userIds = [str(um["userId"]) for um in userMandates] + # Get users for specific mandate using getUsersByMandate + result = appInterface.getUsersByMandate(str(context.mandateId), paginationParams) - # Get all users and filter by mandate membership - allUsers = appInterface.getUsers() - users = [u for u in allUsers if str(u.id) in userIds] + # getUsersByMandate returns PaginatedResult if pagination was provided + if paginationParams and hasattr(result, 'items'): + return PaginatedResponse( + items=result.items, + pagination=PaginationMetadata( + currentPage=result.currentPage, + pageSize=result.pageSize, + totalItems=result.totalItems, + totalPages=result.totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + else: + # No pagination - result is a list + users = result if isinstance(result, list) else result.items if hasattr(result, 'items') else [] + return PaginatedResponse( + items=users, + pagination=None + ) + elif context.isSysAdmin: + # SysAdmin without mandateId sees all users + # Get all users directly from database using UserInDB (the actual database model) + from modules.datamodels.datamodelUam import UserInDB + allUsers = appInterface.db.getRecordset(UserInDB) + # Convert to User objects, filtering out password hash and database-specific fields + users = [] + for u in allUsers: + cleanedUser = {k: v for k, v in u.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} + # Ensure roleLabels is always a list + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + users.append(User(**cleanedUser)) - # Apply pagination manually if needed if paginationParams: totalItems = len(users) import math @@ -105,33 +132,6 @@ async def get_users( items=users, pagination=None ) - elif context.isSysAdmin: - # SysAdmin without mandateId sees all users - result = appInterface.getUsers() - if paginationParams: - totalItems = len(result) - import math - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - paginatedUsers = result[startIdx:endIdx] - - return PaginatedResponse( - items=paginatedUsers, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters - ) - ) - else: - return PaginatedResponse( - items=result, - pagination=None - ) else: # Non-SysAdmin without mandateId - should not happen (getRequestContext enforces) raise HTTPException( diff --git a/modules/routes/routeFeatureChatDynamic.py b/modules/routes/routeFeatureChatDynamic.py index d6f53d84..ed1fd9f3 100644 --- a/modules/routes/routeFeatureChatDynamic.py +++ b/modules/routes/routeFeatureChatDynamic.py @@ -16,7 +16,7 @@ from modules.auth import limiter, getRequestContext, RequestContext import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot # Import models -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum # Import workflow control functions from modules.features.workflow import chatStart, chatStop diff --git a/modules/routes/routeFeatureChatbot.py b/modules/routes/routeFeatureChatbot.py index 20b90876..b5b80e2e 100644 --- a/modules/routes/routeFeatureChatbot.py +++ b/modules/routes/routeFeatureChatbot.py @@ -22,7 +22,7 @@ import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChatbot import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse # Import chatbot feature diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 4d724218..01e33eac 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -89,6 +89,212 @@ async def listFeatures( ) +# ============================================================================= +# My Feature Instances (No mandate context needed) +# IMPORTANT: Must be before /{featureCode} to avoid route matching conflict +# ============================================================================= + +class FeaturesMyResponse(BaseModel): + """Hierarchical response for GET /features/my""" + mandates: List[Dict[str, Any]] + + +@router.get("/my", response_model=FeaturesMyResponse) +@limiter.limit("60/minute") +async def getMyFeatureInstances( + request: Request, + context: RequestContext = Depends(getRequestContext) +) -> FeaturesMyResponse: + """ + Get all feature instances the current user has access to. + + Returns hierarchical structure: mandates -> features -> instances -> permissions + This endpoint does not require X-Mandate-Id header. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Get all feature accesses for this user + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + + if not featureAccesses: + return FeaturesMyResponse(mandates=[]) + + # Build hierarchical structure: mandate -> feature -> instances + mandatesMap: Dict[str, Dict[str, Any]] = {} + featuresMap: Dict[str, Dict[str, Any]] = {} # key: mandateId_featureCode + + for access in featureAccesses: + if not access.enabled: + continue + + instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) + if not instance or not instance.enabled: + continue + + # Get mandate info + mandateId = str(instance.mandateId) + if mandateId not in mandatesMap: + mandate = rootInterface.getMandate(mandateId) + if mandate: + mandatesMap[mandateId] = { + "id": mandateId, + "name": mandate.name if hasattr(mandate, 'name') else mandateId, + "code": mandate.code if hasattr(mandate, 'code') else None, + "features": [] + } + else: + mandatesMap[mandateId] = { + "id": mandateId, + "name": mandateId, + "code": None, + "features": [] + } + + # Get feature info + featureKey = f"{mandateId}_{instance.featureCode}" + if featureKey not in featuresMap: + feature = featureInterface.getFeature(instance.featureCode) + featuresMap[featureKey] = { + "code": instance.featureCode, + "label": feature.label if feature and hasattr(feature, 'label') else {"de": instance.featureCode, "en": instance.featureCode}, + "icon": feature.icon if feature and hasattr(feature, 'icon') else "folder", + "instances": [], + "_mandateId": mandateId # Temporary for grouping + } + + # Get user's role in this instance + userRole = _getUserRoleInInstance(rootInterface, str(context.user.id), str(instance.id)) + + # Get permissions for this instance + permissions = _getInstancePermissions(rootInterface, str(context.user.id), str(instance.id)) + + # Add instance to feature + featuresMap[featureKey]["instances"].append({ + "id": str(instance.id), + "featureCode": instance.featureCode, + "mandateId": mandateId, + "mandateName": mandatesMap[mandateId]["name"], + "instanceLabel": instance.label, + "userRole": userRole, + "permissions": permissions + }) + + # Build final structure + for featureKey, featureData in featuresMap.items(): + mandateId = featureData.pop("_mandateId") + mandatesMap[mandateId]["features"].append(featureData) + + return FeaturesMyResponse(mandates=list(mandatesMap.values())) + + except Exception as e: + logger.error(f"Error getting user's feature instances: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get feature instances: {str(e)}" + ) + + +def _getUserRoleInInstance(rootInterface, userId: str, instanceId: str) -> str: + """Get the user's primary role label in a feature instance.""" + try: + from modules.datamodels.datamodelRbac import UserRole, Role + + # Get user-role assignments for this instance + userRoles = rootInterface.db.getRecordset( + UserRole, + recordFilter={"userId": userId} + ) + + for ur in userRoles: + roleId = ur.get("roleId") + if roleId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles and str(roles[0].get("featureInstanceId")) == instanceId: + return roles[0].get("roleLabel", "user") + + return "user" # Default + except Exception as e: + logger.debug(f"Error getting user role: {e}") + return "user" + + +def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict[str, Any]: + """Get summarized permissions for a user in an instance.""" + # Default permissions structure + permissions = { + "tables": {}, + "views": {}, + "fields": {} + } + + try: + from modules.datamodels.datamodelRbac import UserRole, Role, RolePermission + + # Get user's roles for this instance + userRoles = rootInterface.db.getRecordset(UserRole, recordFilter={"userId": userId}) + roleIds = [] + + for ur in userRoles: + roleId = ur.get("roleId") + if roleId: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles and str(roles[0].get("featureInstanceId")) == instanceId: + roleIds.append(roleId) + + if not roleIds: + return permissions + + # Get permissions for all roles + for roleId in roleIds: + rolePerms = rootInterface.db.getRecordset( + RolePermission, + recordFilter={"roleId": roleId} + ) + + for perm in rolePerms: + tableName = perm.get("tableName", "") + if tableName: + if tableName not in permissions["tables"]: + permissions["tables"][tableName] = { + "view": False, + "read": "n", + "create": "n", + "update": "n", + "delete": "n" + } + + # Merge permissions (highest wins) + current = permissions["tables"][tableName] + current["view"] = current["view"] or perm.get("canView", False) + current["read"] = _mergeAccessLevel(current["read"], perm.get("readLevel", "n")) + current["create"] = _mergeAccessLevel(current["create"], perm.get("createLevel", "n")) + current["update"] = _mergeAccessLevel(current["update"], perm.get("updateLevel", "n")) + current["delete"] = _mergeAccessLevel(current["delete"], perm.get("deleteLevel", "n")) + + viewName = perm.get("viewName", "") + if viewName: + permissions["views"][viewName] = permissions["views"].get(viewName, False) or perm.get("canAccess", False) + + return permissions + + except Exception as e: + logger.debug(f"Error getting instance permissions: {e}") + return permissions + + +def _mergeAccessLevel(current: str, new: str) -> str: + """Merge two access levels, returning the highest.""" + levels = {"n": 0, "m": 1, "g": 2, "a": 3} + currentLevel = levels.get(current, 0) + newLevel = levels.get(new, 0) + + if newLevel > currentLevel: + return new + return current + + @router.get("/{featureCode}", response_model=Dict[str, Any]) @limiter.limit("60/minute") async def getFeature( @@ -539,55 +745,6 @@ async def createTemplateRole( ) -# ============================================================================= -# My Feature Instances (No mandate context needed) -# ============================================================================= - -@router.get("/my", response_model=List[Dict[str, Any]]) -@limiter.limit("60/minute") -async def getMyFeatureInstances( - request: Request, - context: RequestContext = Depends(getRequestContext) -) -> List[Dict[str, Any]]: - """ - Get all feature instances the current user has access to. - - Returns instances across all mandates the user is member of. - This endpoint does not require X-Mandate-Id header. - """ - try: - rootInterface = getRootInterface() - - # Get all feature accesses for this user - featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) - - if not featureAccesses: - return [] - - featureInterface = getFeatureInterface(rootInterface.db) - result = [] - - for access in featureAccesses: - if not access.enabled: - continue - - instance = featureInterface.getFeatureInstance(str(access.featureInstanceId)) - if instance and instance.enabled: - result.append({ - **instance.model_dump(), - "accessId": str(access.id) - }) - - return result - - except Exception as e: - logger.error(f"Error getting user's feature instances: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to get feature instances: {str(e)}" - ) - - # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 1c7d9e80..ab9e1ff6 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -19,7 +19,7 @@ from modules.interfaces.interfaceDbChatbot import getInterface from modules.interfaces.interfaceRbac import getRecordsetWithRBAC # Import models -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( ChatWorkflow, ChatMessage, ChatLog, diff --git a/modules/services/__init__.py b/modules/services/__init__.py index b75b2454..fb4e6512 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -3,7 +3,7 @@ from typing import Any, Optional from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow class PublicService: """Lightweight proxy exposing only public callable attributes of a target. diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 55bd2544..cd86c6a8 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -6,7 +6,7 @@ import re import time import base64 from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChatbot import PromptPlaceholder, ChatDocument +from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py index ec6a26d2..a866f68f 100644 --- a/modules/services/serviceAi/subContentExtraction.py +++ b/modules/services/serviceAi/subContentExtraction.py @@ -14,7 +14,7 @@ import logging import base64 from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py index e90ecfeb..821851a4 100644 --- a/modules/services/serviceAi/subDocumentIntents.py +++ b/modules/services/serviceAi/subDocumentIntents.py @@ -12,7 +12,7 @@ import json import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelExtraction import DocumentIntent from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 0c82929d..137dcd05 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChatbot import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 64678f54..13739dea 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -11,7 +11,7 @@ import json from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult, DocumentIntent -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index adc0ea78..a49b78c7 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -6,7 +6,7 @@ import base64 import traceback from typing import Any, Dict, List, Optional, Callable from modules.datamodels.datamodelDocument import RenderedDocument -from modules.datamodels.datamodelChatbot import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.services.serviceGeneration.subDocumentUtility import ( getFileExtension, getMimeTypeFromExtension, diff --git a/modules/shared/dbMultiTenantOptimizations.py b/modules/shared/dbMultiTenantOptimizations.py index 38f154a5..3e056179 100644 --- a/modules/shared/dbMultiTenantOptimizations.py +++ b/modules/shared/dbMultiTenantOptimizations.py @@ -21,6 +21,22 @@ from typing import Optional, List logger = logging.getLogger(__name__) +def _getConnection(dbConnector): + """Get a connection from the DatabaseConnector. + + Ensures the connection is alive and returns it. + Commits any pending transaction first to avoid blocking. + """ + dbConnector._ensure_connection() + conn = dbConnector.connection + # Commit any pending transaction to avoid blocking + try: + conn.commit() + except Exception: + pass # Ignore if nothing to commit + return conn + + # ============================================================================= # Index Definitions # ============================================================================= @@ -144,28 +160,45 @@ def applyMultiTenantOptimizations(dbConnector, tables: Optional[List[str]] = Non try: # Get a connection from the connector - conn = dbConnector._get_connection() - conn.autocommit = True + conn = _getConnection(dbConnector) - with conn.cursor() as cursor: - # Apply indexes - results["indexesCreated"] = _applyIndexes(cursor, tables) - - # Apply foreign keys - results["foreignKeysCreated"] = _applyForeignKeys(cursor, tables) - - # Apply immutable triggers - results["triggersCreated"] = _applyImmutableTriggers(cursor, tables) + # Save and set autocommit state + try: + originalAutocommit = conn.autocommit + except Exception: + originalAutocommit = False - logger.info( - f"Multi-tenant optimizations applied: " - f"{results['indexesCreated']} indexes, " - f"{results['triggersCreated']} triggers, " - f"{results['foreignKeysCreated']} foreign keys" - ) + try: + conn.autocommit = True + except Exception as autoErr: + logger.debug(f"Could not set autocommit: {autoErr}") + + try: + with conn.cursor() as cursor: + # Apply indexes + results["indexesCreated"] = _applyIndexes(cursor, tables) + + # Apply foreign keys + results["foreignKeysCreated"] = _applyForeignKeys(cursor, tables) + + # Apply immutable triggers + results["triggersCreated"] = _applyImmutableTriggers(cursor, tables) + + logger.info( + f"Multi-tenant optimizations applied: " + f"{results['indexesCreated']} indexes, " + f"{results['triggersCreated']} triggers, " + f"{results['foreignKeysCreated']} foreign keys" + ) + finally: + # Restore original autocommit state + try: + conn.autocommit = originalAutocommit + except Exception: + pass except Exception as e: - logger.error(f"Error applying multi-tenant optimizations: {e}") + logger.error(f"Error applying multi-tenant optimizations: {type(e).__name__}: {e}") results["errors"].append(str(e)) return results @@ -174,11 +207,15 @@ def applyMultiTenantOptimizations(dbConnector, tables: Optional[List[str]] = Non def applyIndexesOnly(dbConnector, tables: Optional[List[str]] = None) -> int: """Apply only indexes (lighter operation, safe for frequent calls).""" try: - conn = dbConnector._get_connection() + conn = _getConnection(dbConnector) + originalAutocommit = conn.autocommit conn.autocommit = True - with conn.cursor() as cursor: - return _applyIndexes(cursor, tables) + try: + with conn.cursor() as cursor: + return _applyIndexes(cursor, tables) + finally: + conn.autocommit = originalAutocommit except Exception as e: logger.error(f"Error applying indexes: {e}") return 0 @@ -194,9 +231,13 @@ def _tableExists(cursor, tableName: str) -> bool: SELECT EXISTS ( SELECT FROM information_schema.tables WHERE table_name = %s - ) + ) AS exists """, (tableName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + # Handle both dict (RealDictCursor) and tuple results + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _indexExists(cursor, indexName: str) -> bool: @@ -205,9 +246,12 @@ def _indexExists(cursor, indexName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_indexes WHERE indexname = %s - ) + ) AS exists """, (indexName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _constraintExists(cursor, constraintName: str) -> bool: @@ -216,9 +260,12 @@ def _constraintExists(cursor, constraintName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_constraint WHERE conname = %s - ) + ) AS exists """, (constraintName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _triggerExists(cursor, triggerName: str) -> bool: @@ -227,9 +274,12 @@ def _triggerExists(cursor, triggerName: str) -> bool: SELECT EXISTS ( SELECT FROM pg_trigger WHERE tgname = %s - ) + ) AS exists """, (triggerName,)) - return cursor.fetchone()[0] + row = cursor.fetchone() + if isinstance(row, dict): + return row.get('exists', False) + return row[0] if row else False def _applyIndexes(cursor, tables: Optional[List[str]]) -> int: @@ -390,7 +440,7 @@ def getOptimizationStatus(dbConnector) -> dict: } try: - conn = dbConnector._get_connection() + conn = _getConnection(dbConnector) with conn.cursor() as cursor: # Check regular indexes for tableName, indexName, _ in _INDEXES: diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py index a3f45261..39d6e16f 100644 --- a/modules/workflows/methods/methodAi/actions/convertDocument.py +++ b/modules/workflows/methods/methodAi/actions/convertDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 77cb361f..4f9bbd21 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 6c509a9e..65e95a32 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index bddeb252..f804c0b9 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -5,7 +5,7 @@ import logging import time import json from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallOptions from modules.datamodels.datamodelExtraction import ContentPart diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py index 806679df..e32c1965 100644 --- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py +++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py index 96f2609c..bb6f8437 100644 --- a/modules/workflows/methods/methodAi/actions/translateDocument.py +++ b/modules/workflows/methods/methodAi/actions/translateDocument.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult +from modules.datamodels.datamodelChat import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 4c5d8314..62b43bce 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -5,7 +5,7 @@ import logging import time import re from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py index c6e6d560..ff7e896f 100644 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py @@ -11,7 +11,7 @@ import json import time from typing import Dict, Any from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.connectors.connectorPreprocessor import PreprocessorConnector logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index cccad45d..5b90ce13 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy, ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py index fedbc46b..9991285b 100644 --- a/modules/workflows/methods/methodContext/actions/getDocumentIndex.py +++ b/modules/workflows/methods/methodContext/actions/getDocumentIndex.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodContext/actions/neutralizeData.py b/modules/workflows/methods/methodContext/actions/neutralizeData.py index 0b646251..8e3b7185 100644 --- a/modules/workflows/methods/methodContext/actions/neutralizeData.py +++ b/modules/workflows/methods/methodContext/actions/neutralizeData.py @@ -4,7 +4,7 @@ import logging import time from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelDocref import DocumentReferenceList from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart diff --git a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py index 015eb1e3..2f011a25 100644 --- a/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py +++ b/modules/workflows/methods/methodContext/actions/triggerPreprocessingServer.py @@ -5,7 +5,7 @@ import logging import json import aiohttp from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/connectJira.py b/modules/workflows/methods/methodJira/actions/connectJira.py index f00192f6..45b60cad 100644 --- a/modules/workflows/methods/methodJira/actions/connectJira.py +++ b/modules/workflows/methods/methodJira/actions/connectJira.py @@ -5,7 +5,7 @@ import logging import json import uuid from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createCsvContent.py b/modules/workflows/methods/methodJira/actions/createCsvContent.py index 1e44b1cb..cbec7960 100644 --- a/modules/workflows/methods/methodJira/actions/createCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/createCsvContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import StringIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/createExcelContent.py b/modules/workflows/methods/methodJira/actions/createExcelContent.py index afa0c5fc..631795b3 100644 --- a/modules/workflows/methods/methodJira/actions/createExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/createExcelContent.py @@ -9,7 +9,7 @@ import csv as csv_module from io import BytesIO from datetime import datetime, UTC from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py index 6e6106b0..55d99654 100644 --- a/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py +++ b/modules/workflows/methods/methodJira/actions/exportTicketsAsJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py index d72e3d55..b997889e 100644 --- a/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py +++ b/modules/workflows/methods/methodJira/actions/importTicketsFromJson.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/mergeTicketData.py b/modules/workflows/methods/methodJira/actions/mergeTicketData.py index 49ddece2..2bd7ab74 100644 --- a/modules/workflows/methods/methodJira/actions/mergeTicketData.py +++ b/modules/workflows/methods/methodJira/actions/mergeTicketData.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseCsvContent.py b/modules/workflows/methods/methodJira/actions/parseCsvContent.py index 24c097e8..bbdc2cc7 100644 --- a/modules/workflows/methods/methodJira/actions/parseCsvContent.py +++ b/modules/workflows/methods/methodJira/actions/parseCsvContent.py @@ -6,7 +6,7 @@ import json import io import pandas as pd from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodJira/actions/parseExcelContent.py b/modules/workflows/methods/methodJira/actions/parseExcelContent.py index adfe13ea..5ac4e548 100644 --- a/modules/workflows/methods/methodJira/actions/parseExcelContent.py +++ b/modules/workflows/methods/methodJira/actions/parseExcelContent.py @@ -6,7 +6,7 @@ import json import pandas as pd from io import BytesIO from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py index 06f26e89..59604896 100644 --- a/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py +++ b/modules/workflows/methods/methodOutlook/actions/composeAndDraftEmailWithContext.py @@ -6,7 +6,7 @@ import json import base64 import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/readEmails.py b/modules/workflows/methods/methodOutlook/actions/readEmails.py index 4ff700ca..2d325d9f 100644 --- a/modules/workflows/methods/methodOutlook/actions/readEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/readEmails.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/searchEmails.py b/modules/workflows/methods/methodOutlook/actions/searchEmails.py index 4531859f..f8831d59 100644 --- a/modules/workflows/methods/methodOutlook/actions/searchEmails.py +++ b/modules/workflows/methods/methodOutlook/actions/searchEmails.py @@ -5,7 +5,7 @@ import logging import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py index da9f8cd4..9b7fb011 100644 --- a/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py +++ b/modules/workflows/methods/methodOutlook/actions/sendDraftEmail.py @@ -6,7 +6,7 @@ import time import json import requests from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py index 05997512..a4bf18b6 100644 --- a/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py +++ b/modules/workflows/methods/methodSharepoint/actions/analyzeFolderUsage.py @@ -6,7 +6,7 @@ import time import json from datetime import datetime, timezone, timedelta from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/copyFile.py b/modules/workflows/methods/methodSharepoint/actions/copyFile.py index 287612ff..f149e482 100644 --- a/modules/workflows/methods/methodSharepoint/actions/copyFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/copyFile.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py index e6c2a276..c64a6637 100644 --- a/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/downloadFileByPath.py @@ -6,7 +6,7 @@ import json import base64 import os from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py index 4eac8544..722dbc99 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py +++ b/modules/workflows/methods/methodSharepoint/actions/findDocumentPath.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py index a9b837aa..62b6dd94 100644 --- a/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py +++ b/modules/workflows/methods/methodSharepoint/actions/findSiteByUrl.py @@ -4,7 +4,7 @@ import logging import json from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py index d0838633..318271c3 100644 --- a/modules/workflows/methods/methodSharepoint/actions/listDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/listDocuments.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py index eaf3254f..73cdb730 100644 --- a/modules/workflows/methods/methodSharepoint/actions/readDocuments.py +++ b/modules/workflows/methods/methodSharepoint/actions/readDocuments.py @@ -6,7 +6,7 @@ import time import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py index ddce6206..e9361853 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadDocument.py @@ -6,7 +6,7 @@ import time import json import urllib.parse from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py index 85d7b123..1f469b80 100644 --- a/modules/workflows/methods/methodSharepoint/actions/uploadFile.py +++ b/modules/workflows/methods/methodSharepoint/actions/uploadFile.py @@ -5,7 +5,7 @@ import logging import json import base64 from typing import Dict, Any -from modules.datamodels.datamodelChatbot import ActionResult, ActionDocument +from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/actionExecutor.py b/modules/workflows/processing/core/actionExecutor.py index 7bde4da7..0e4d6ee4 100644 --- a/modules/workflows/processing/core/actionExecutor.py +++ b/modules/workflows/processing/core/actionExecutor.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import ActionResult, ActionItem, TaskStep -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ActionResult, ActionItem, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.methodDiscovery import methods from modules.workflows.processing.shared.stateTools import checkWorkflowStopped diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index 0daac228..a4ae05e9 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -5,8 +5,8 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChatbot import TaskPlan, TaskStep, ActionResult, ReviewResult -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import TaskPlan, TaskStep, ActionResult, ReviewResult +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.shared.stateTools import checkWorkflowStopped logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 4b1fcad5..0fac427c 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -6,7 +6,7 @@ import json import logging from typing import Dict, Any -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.shared.promptGenerationTaskplan import ( generateTaskPlanningPrompt @@ -51,7 +51,7 @@ class TaskPlanner: # Analyze user intent to obtain cleaned user objective for planning # SKIP intent analysis for AUTOMATION mode - it uses predefined JSON plans - from modules.datamodels.datamodelChatbot import WorkflowModeEnum + from modules.datamodels.datamodelChat import WorkflowModeEnum workflowMode = getattr(workflow, 'workflowMode', None) skipIntentionAnalysis = (workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION) diff --git a/modules/workflows/processing/modes/modeAutomation.py b/modules/workflows/processing/modes/modeAutomation.py index 85f1f824..e3131939 100644 --- a/modules/workflows/processing/modes/modeAutomation.py +++ b/modules/workflows/processing/modes/modeAutomation.py @@ -7,11 +7,11 @@ import json import logging import uuid from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, TaskPlan, ActionResult ) -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp diff --git a/modules/workflows/processing/modes/modeBase.py b/modules/workflows/processing/modes/modeBase.py index e8837d65..770c868a 100644 --- a/modules/workflows/processing/modes/modeBase.py +++ b/modules/workflows/processing/modes/modeBase.py @@ -6,8 +6,8 @@ from abc import ABC, abstractmethod import logging from typing import List, Dict, Any -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskResult, ActionItem -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskResult, ActionItem +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.core.taskPlanner import TaskPlanner from modules.workflows.processing.core.actionExecutor import ActionExecutor from modules.workflows.processing.core.messageCreator import MessageCreator diff --git a/modules/workflows/processing/modes/modeDynamic.py b/modules/workflows/processing/modes/modeDynamic.py index b821511b..f7754eab 100644 --- a/modules/workflows/processing/modes/modeDynamic.py +++ b/modules/workflows/processing/modes/modeDynamic.py @@ -9,11 +9,11 @@ import re import time from datetime import datetime, timezone from typing import List, Dict, Any -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( TaskStep, TaskContext, TaskResult, ActionItem, TaskStatus, ActionResult, Observation, ObservationPreview, ReviewResult ) -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.shared.timeUtils import parseTimestamp @@ -893,7 +893,7 @@ class DynamicMode(BaseMode): async def _refineDecide(self, context: TaskContext, observation: Observation) -> ReviewResult: """Refine: decide continue or stop, with reason""" # Create proper ReviewContext for extractReviewContent - from modules.datamodels.datamodelChatbot import ReviewContext + from modules.datamodels.datamodelChat import ReviewContext # Convert observation to dict for extractReviewContent (temporary compatibility) observationDict = { 'success': observation.success, @@ -1042,7 +1042,7 @@ class DynamicMode(BaseMode): # Parse response using structured parsing with ReviewResult model from modules.shared.jsonUtils import parseJsonWithModel - from modules.datamodels.datamodelChatbot import ReviewResult + from modules.datamodels.datamodelChat import ReviewResult if not resp: return ReviewResult( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index b0186be9..1cdf0d53 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -5,7 +5,7 @@ import logging from typing import List, Optional -from modules.datamodels.datamodelChatbot import TaskStep, ActionResult, Observation +from modules.datamodels.datamodelChat import TaskStep, ActionResult, Observation logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 30e9af4d..a6e3a78a 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -348,7 +348,7 @@ def extractReviewContent(context: Any) -> str: elif hasattr(context, 'observation') and context.observation: # For observation data, show full content but handle documents specially # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChatbot import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(context.observation, Observation): # Convert Pydantic model to dict @@ -371,7 +371,7 @@ def extractReviewContent(context: Any) -> str: # For observation data in stepResult, show full content but handle documents specially observation = context.stepResult['observation'] # Handle both Pydantic Observation model and dict format - from modules.datamodels.datamodelChatbot import Observation + from modules.datamodels.datamodelChat import Observation if isinstance(observation, Observation): # Convert Pydantic model to dict diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index 10932529..31878033 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -7,7 +7,7 @@ Handles prompt templates for dynamic mode action handling. import json from typing import Any, List -from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractUserLanguage, diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index e1d767c4..11a54ca1 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -7,7 +7,7 @@ Handles prompt templates and extraction functions for task planning phase. import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChatbot import PromptBundle, PromptPlaceholder +from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder from modules.workflows.processing.shared.placeholderFactory import ( extractUserPrompt, extractAvailableDocumentsSummary, diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index 317a6cb7..9c9d6c84 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -6,9 +6,9 @@ import logging import json from typing import Dict, Any, Optional, List, TYPE_CHECKING -from modules.datamodels import datamodelChatbot -from modules.datamodels.datamodelChatbot import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage -from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum +from modules.datamodels import datamodelChat +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, ActionResult, ActionDocument, ChatDocument, ChatMessage +from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeDynamic import DynamicMode from modules.workflows.processing.modes.modeAutomation import AutomationMode @@ -102,7 +102,7 @@ class WorkflowProcessor: self.services.chat.progressLogFinish(operationId, False) raise - async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChatbot.TaskResult: + async def executeTask(self, taskStep: TaskStep, workflow: ChatWorkflow, context: TaskContext) -> datamodelChat.TaskResult: """Execute a task step using the appropriate mode""" import time @@ -494,7 +494,7 @@ class WorkflowProcessor: # Create ActionResult with response # For fast path, we create a simple text document with the response - from modules.datamodels.datamodelChatbot import ActionDocument + from modules.datamodels.datamodelChat import ActionDocument responseDoc = ActionDocument( documentName="fast_path_response.txt", @@ -626,7 +626,7 @@ class WorkflowProcessor: ChatMessage with persisted documents """ try: - from modules.datamodels.datamodelChatbot import ChatMessage, ChatDocument, ActionDocument + from modules.datamodels.datamodelChat import ChatMessage, ChatDocument, ActionDocument from modules.workflows.processing.shared.stateTools import checkWorkflowStopped # Check workflow status diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index e6ecdbbd..a9b656eb 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,14 +6,14 @@ import uuid import asyncio import json -from modules.datamodels.datamodelChatbot import ( +from modules.datamodels.datamodelChat import ( UserInputRequest, ChatMessage, ChatWorkflow, ChatDocument, WorkflowModeEnum ) -from modules.datamodels.datamodelChatbot import TaskContext +from modules.datamodels.datamodelChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor from modules.workflows.processing.shared.stateTools import WorkflowStoppedException, checkWorkflowStopped @@ -606,7 +606,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -792,7 +792,7 @@ The following is the user's original input message. Analyze intent, normalize th # Collect file info fileInfo = self.services.chat.getFileInfo(fileItem.id) - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument doc = ChatDocument( fileId=fileItem.id, fileName=fileInfo.get("fileName", fileName) if fileInfo else fileName, @@ -921,7 +921,7 @@ The following is the user's original input message. Analyze intent, normalize th # Persist task result for cross-task/round document references # Convert ChatTaskResult to WorkflowTaskResult for persistence from modules.datamodels.datamodelWorkflow import TaskResult as WorkflowTaskResult - from modules.datamodels.datamodelChatbot import ActionResult + from modules.datamodels.datamodelChat import ActionResult # Get final ActionResult from task execution (last action result) finalActionResult = None diff --git a/tests/functional/test02_ai_models.py b/tests/functional/test02_ai_models.py index 94eb6158..12a374f8 100644 --- a/tests/functional/test02_ai_models.py +++ b/tests/functional/test02_ai_models.py @@ -85,7 +85,7 @@ class AIModelsTester: self.services.extraction = ExtractionService(self.services) # Create a minimal workflow context - from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid self.services.currentWorkflow = ChatWorkflow( diff --git a/tests/functional/test03_ai_operations.py b/tests/functional/test03_ai_operations.py index dd5d68e3..05a3f34b 100644 --- a/tests/functional/test03_ai_operations.py +++ b/tests/functional/test03_ai_operations.py @@ -18,7 +18,7 @@ if _gateway_path not in sys.path: sys.path.insert(0, _gateway_path) from modules.datamodels.datamodelAi import OperationTypeEnum -from modules.datamodels.datamodelChatbot import ChatWorkflow, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelChat import ChatWorkflow, ChatDocument, WorkflowModeEnum from modules.datamodels.datamodelUam import User @@ -174,7 +174,7 @@ class MethodAiOperationsTester: imageData = f.read() # Create a ChatDocument - from modules.datamodels.datamodelChatbot import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument import uuid testImageDoc = ChatDocument( @@ -186,7 +186,7 @@ class MethodAiOperationsTester: ) # Create a message with this document - from modules.datamodels.datamodelChatbot import ChatMessage + from modules.datamodels.datamodelChat import ChatMessage import time testMessage = ChatMessage( diff --git a/tests/functional/test04_ai_behavior.py b/tests/functional/test04_ai_behavior.py index 478e9baf..9da22d66 100644 --- a/tests/functional/test04_ai_behavior.py +++ b/tests/functional/test04_ai_behavior.py @@ -42,7 +42,7 @@ class AIBehaviorTester: logging.getLogger().setLevel(logging.DEBUG) # Create and save workflow in database using the interface - from modules.datamodels.datamodelChatbot import ChatWorkflow, WorkflowModeEnum + from modules.datamodels.datamodelChat import ChatWorkflow, WorkflowModeEnum import uuid import time import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test05_workflow_with_documents.py b/tests/functional/test05_workflow_with_documents.py index 3a0cf2a3..7beaeec4 100644 --- a/tests/functional/test05_workflow_with_documents.py +++ b/tests/functional/test05_workflow_with_documents.py @@ -20,7 +20,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test06_workflow_prompt_variations.py b/tests/functional/test06_workflow_prompt_variations.py index 784b1756..698c9698 100644 --- a/tests/functional/test06_workflow_prompt_variations.py +++ b/tests/functional/test06_workflow_prompt_variations.py @@ -22,7 +22,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test09_document_generation_formats.py b/tests/functional/test09_document_generation_formats.py index 9c96d95f..d9c4d9b8 100644 --- a/tests/functional/test09_document_generation_formats.py +++ b/tests/functional/test09_document_generation_formats.py @@ -21,7 +21,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test10_document_generation_formats.py b/tests/functional/test10_document_generation_formats.py index a0b744f4..45a364ce 100644 --- a/tests/functional/test10_document_generation_formats.py +++ b/tests/functional/test10_document_generation_formats.py @@ -21,7 +21,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/functional/test11_code_generation_formats.py b/tests/functional/test11_code_generation_formats.py index e1331cd1..6d1735ad 100644 --- a/tests/functional/test11_code_generation_formats.py +++ b/tests/functional/test11_code_generation_formats.py @@ -23,7 +23,7 @@ if _gateway_path not in sys.path: # Import the service initialization from modules.services import getInterface as getServices -from modules.datamodels.datamodelChatbot import UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest, WorkflowModeEnum from modules.datamodels.datamodelUam import User from modules.features.workflow import chatStart import modules.interfaces.interfaceDbChatbot as interfaceDbChatbot diff --git a/tests/integration/workflows/test_workflow_execution.py b/tests/integration/workflows/test_workflow_execution.py index 9409a9e6..a2b69576 100644 --- a/tests/integration/workflows/test_workflow_execution.py +++ b/tests/integration/workflows/test_workflow_execution.py @@ -10,7 +10,7 @@ import pytest import uuid from unittest.mock import Mock, AsyncMock, patch -from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference, DocumentItemReference diff --git a/tests/unit/workflows/test_state_management.py b/tests/unit/workflows/test_state_management.py index b91aa1e7..ae502397 100644 --- a/tests/unit/workflows/test_state_management.py +++ b/tests/unit/workflows/test_state_management.py @@ -9,7 +9,7 @@ Tests state increment methods, helper methods, and updateFromSelection. import pytest import uuid -from modules.datamodels.datamodelChatbot import ChatWorkflow, TaskContext, TaskStep +from modules.datamodels.datamodelChat import ChatWorkflow, TaskContext, TaskStep from modules.datamodels.datamodelWorkflow import ActionDefinition diff --git a/tests/validation/test_architecture_validation.py b/tests/validation/test_architecture_validation.py index 25a5af8e..09f6e92c 100644 --- a/tests/validation/test_architecture_validation.py +++ b/tests/validation/test_architecture_validation.py @@ -15,7 +15,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from modules.datamodels.datamodelWorkflow import ActionDefinition, AiResponse from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentListReference -from modules.datamodels.datamodelChatbot import ChatWorkflow +from modules.datamodels.datamodelChat import ChatWorkflow from modules.shared.jsonUtils import parseJsonWithModel diff --git a/tool_db_adapt_to_models.py b/tool_db_adapt_to_models.py new file mode 100644 index 00000000..85c6e8fc --- /dev/null +++ b/tool_db_adapt_to_models.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +""" +Datenbank-Anpassung an Pydantic-Modelle. + +Einfaches Script das: +1. Fehlende Felder in DB ergänzt (gemäss Pydantic-Modellen) +2. Falsche Datentypen korrigiert (Daten werden ggf. gelöscht) +3. Spezialfall: UserInDB.privilege → roleLabels migriert + +Verwendung: + python tool_db_adapt_to_models.py [--dry-run] [--db ] +""" + +import os +import sys +import json +import argparse +import logging +from pathlib import Path +from typing import Dict, List, Any, Optional + +# Gateway-Pfad setzen +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent +sys.path.insert(0, str(gatewayPath)) +os.chdir(str(gatewayPath)) + +# Logging ZUERST konfigurieren +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + force=True # Überschreibt bestehende Config +) +logger = logging.getLogger(__name__) + +import psycopg2 +import psycopg2.extras +from modules.shared.configuration import APP_CONFIG + +# Datenbank-Konfiguration: DB-Name → (Config-Prefix, Pydantic-Modelle) +DATABASE_CONFIG = { + "poweron_app": ("DB_APP", ["datamodelUam", "datamodelRbac", "datamodelSecurity"]), + "poweron_chat": ("DB_CHAT", ["datamodelChat"]), + "poweron_management": ("DB_MANAGEMENT", ["datamodelWorkflow", "datamodelFiles"]), +} + +# Python-Typ → PostgreSQL-Typ Mapping +TYPE_MAPPING = { + "str": "text", + "int": "integer", + "float": "double precision", + "bool": "boolean", + "list": "jsonb", + "dict": "jsonb", + "List": "jsonb", + "Dict": "jsonb", + "Optional": None, # Wird separat behandelt + "datetime": "timestamp", + "EmailStr": "text", + "UUID": "uuid", +} + + +def _getDbConnection(dbName: str): + """Verbindet mit einer Datenbank über APP_CONFIG.""" + prefix = DATABASE_CONFIG.get(dbName, ("DB", []))[0] + + host = APP_CONFIG.get(f"{prefix}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") + port = APP_CONFIG.get(f"{prefix}_PORT") or APP_CONFIG.get("DB_PORT", "5432") + user = APP_CONFIG.get(f"{prefix}_USER") or APP_CONFIG.get("DB_USER") + password = APP_CONFIG.get(f"{prefix}_PASSWORD_SECRET") or APP_CONFIG.get(f"{prefix}_PASSWORD") or APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD") + + if not user or not password: + logger.error(f"Keine Credentials für {dbName} ({prefix}_*)") + return None + + try: + conn = psycopg2.connect( + host=host, port=int(port), database=dbName, + user=user, password=password, + cursor_factory=psycopg2.extras.RealDictCursor + ) + conn.autocommit = True + return conn + except Exception as e: + logger.error(f"Verbindungsfehler {dbName}: {e}") + return None + + +def _getDbTables(conn) -> Dict[str, Dict[str, str]]: + """Holt alle Tabellen und deren Spalten aus der DB.""" + cur = conn.cursor() + cur.execute(""" + SELECT table_name, column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = 'public' + ORDER BY table_name, ordinal_position + """) + + tables = {} + for row in cur.fetchall(): + tableName = row['table_name'] + if tableName not in tables: + tables[tableName] = {} + tables[tableName][row['column_name']] = { + 'type': row['data_type'], + 'nullable': row['is_nullable'] == 'YES' + } + cur.close() + return tables + + +def _parsePydanticModels(moduleNames: List[str]) -> Dict[str, Dict[str, str]]: + """Parst Pydantic-Modelle aus den angegebenen Modulen.""" + import ast + + models = {} + datamodelsPath = gatewayPath / "modules" / "datamodels" + + for moduleName in moduleNames: + filePath = datamodelsPath / f"{moduleName}.py" + if not filePath.exists(): + logger.warning(f"Modul nicht gefunden: {filePath}") + continue + + with open(filePath, 'r', encoding='utf-8') as f: + tree = ast.parse(f.read()) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + className = node.name + + # Prüfe ob Klasse von BaseModel erbt + isBaseModel = False + for base in node.bases: + if isinstance(base, ast.Name) and base.id == "BaseModel": + isBaseModel = True + break + + if not isBaseModel: + continue + + fields = {} + for item in node.body: + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name): + fieldName = item.target.id + if fieldName.startswith('_'): + continue + + # Typ extrahieren + fieldType = _extractType(item.annotation) + if fieldType: + fields[fieldName] = fieldType + + if fields: + models[className] = fields + + return models + + +def _extractType(annotation) -> Optional[str]: + """Extrahiert den PostgreSQL-Typ aus einer AST-Annotation.""" + import ast + + if isinstance(annotation, ast.Name): + return TYPE_MAPPING.get(annotation.id, "text") + + elif isinstance(annotation, ast.Subscript): + # Optional[X], List[X], Dict[X, Y] + if isinstance(annotation.value, ast.Name): + outerType = annotation.value.id + if outerType == "Optional": + # Rekursiv den inneren Typ holen + if isinstance(annotation.slice, ast.Name): + return TYPE_MAPPING.get(annotation.slice.id, "text") + elif isinstance(annotation.slice, ast.Subscript): + return _extractType(annotation.slice) + return "text" + elif outerType in ("List", "list", "Dict", "dict"): + return "jsonb" + + elif isinstance(annotation, ast.Constant): + return "text" + + return "text" + + +def _adaptTable(conn, tableName: str, modelFields: Dict[str, str], dbColumns: Dict[str, Any], dryRun: bool) -> bool: + """Passt eine Tabelle an das Pydantic-Modell an.""" + cur = conn.cursor() + success = True + + for fieldName, pgType in modelFields.items(): + # pgType kommt direkt aus _extractType (ist bereits PostgreSQL-Typ) + + # Suche Spalte case-insensitive + dbCol = None + actualColName = None + for colName, colInfo in dbColumns.items(): + if colName.lower() == fieldName.lower(): + dbCol = colInfo + actualColName = colName + break + + if dbCol is None: + # Spalte fehlt → hinzufügen + sql = f'ALTER TABLE "{tableName}" ADD COLUMN "{fieldName}" {pgType}' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + else: + try: + cur.execute(sql) + logger.info(f"Spalte hinzugefügt: {tableName}.{fieldName} ({pgType})") + except Exception as e: + logger.error(f"Fehler beim Hinzufügen von {tableName}.{fieldName}: {e}") + success = False + else: + # Spalte existiert → Typ prüfen + currentType = dbCol['type'] + if not _typesCompatible(currentType, pgType): + sql = f'ALTER TABLE "{tableName}" ALTER COLUMN "{actualColName}" TYPE {pgType} USING NULL' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + else: + try: + cur.execute(sql) + logger.info(f"Typ geändert: {tableName}.{actualColName} ({currentType} → {pgType})") + except Exception as e: + logger.error(f"Fehler beim Ändern von {tableName}.{actualColName}: {e}") + success = False + + cur.close() + return success + + +def _typesCompatible(dbType: str, targetType: str) -> bool: + """Prüft ob DB-Typ mit Ziel-Typ kompatibel ist.""" + dbType = dbType.lower() + targetType = targetType.lower() + + # Gleiche Typen + if dbType == targetType: + return True + + # Kompatible Typen + compatiblePairs = [ + ("character varying", "text"), + ("varchar", "text"), + ("integer", "bigint"), + ("real", "double precision"), + ("timestamp without time zone", "timestamp"), + ("timestamp with time zone", "timestamp"), + ] + + for a, b in compatiblePairs: + if (dbType == a and targetType == b) or (dbType == b and targetType == a): + return True + + return False + + +def _migratePrivilegeToRoleLabels(conn, tableName: str, dryRun: bool) -> bool: + """Migriert privilege-Wert nach roleLabels (Spezialfall UserInDB).""" + cur = conn.cursor() + + # Prüfe ob beide Spalten existieren + cur.execute(""" + SELECT column_name FROM information_schema.columns + WHERE table_name = %s AND column_name IN ('privilege', 'roleLabels') + """, (tableName,)) + columns = [row['column_name'] for row in cur.fetchall()] + + if 'privilege' not in columns: + logger.info(f"Spalte 'privilege' existiert nicht in {tableName} - keine Migration nötig") + cur.close() + return True + + if 'roleLabels' not in columns: + # roleLabels erstellen + sql = f'ALTER TABLE "{tableName}" ADD COLUMN "roleLabels" jsonb' + if dryRun: + logger.info(f"[DRY-RUN] {sql}") + cur.close() + return True + else: + cur.execute(sql) + logger.info(f"Spalte roleLabels in {tableName} erstellt") + + # Migriere privilege → roleLabels + cur.execute(f""" + SELECT id, privilege, "roleLabels" FROM "{tableName}" + WHERE privilege IS NOT NULL AND privilege != '' + """) + users = cur.fetchall() + + if not users: + logger.info(f"Keine Einträge mit privilege-Wert in {tableName}") + cur.close() + return True + + logger.info(f"Migriere {len(users)} Einträge: privilege → roleLabels") + + for user in users: + userId = user['id'] + privilege = user['privilege'] + roleLabels = user['roleLabels'] or [] + + if isinstance(roleLabels, str): + try: + roleLabels = json.loads(roleLabels) + except: + roleLabels = [] + + if privilege not in roleLabels: + roleLabels.append(privilege) + + if dryRun: + logger.info(f"[DRY-RUN] UPDATE {tableName} SET roleLabels = {roleLabels} WHERE id = {userId}") + else: + cur.execute( + f'UPDATE "{tableName}" SET "roleLabels" = %s WHERE id = %s', + (json.dumps(roleLabels), userId) + ) + + if not dryRun: + logger.info(f"Migration privilege → roleLabels abgeschlossen") + + cur.close() + return True + + +def runMigration(dbName: str, dryRun: bool) -> bool: + """Führt Migration für eine Datenbank durch.""" + logger.info(f"\n{'='*60}") + logger.info(f"DATENBANK: {dbName}") + logger.info(f"{'='*60}") + + if dbName not in DATABASE_CONFIG: + logger.error(f"Unbekannte Datenbank: {dbName}") + return False + + conn = _getDbConnection(dbName) + if not conn: + return False + + prefix, moduleNames = DATABASE_CONFIG[dbName] + + # DB-Schema laden + dbTables = _getDbTables(conn) + logger.info(f"Tabellen in DB: {', '.join(dbTables.keys()) if dbTables else 'keine'}") + + # Pydantic-Modelle laden + models = _parsePydanticModels(moduleNames) + logger.info(f"Pydantic-Modelle: {', '.join(models.keys()) if models else 'keine'}") + + success = True + + # Jedes Modell mit passender DB-Tabelle abgleichen + for modelName, modelFields in models.items(): + # Finde passende Tabelle (case-insensitive) + tableName = None + tableColumns = None + for dbTable, dbCols in dbTables.items(): + if dbTable.lower() == modelName.lower(): + tableName = dbTable + tableColumns = dbCols + break + + if tableName is None: + logger.warning(f"Keine Tabelle für Modell {modelName} gefunden - übersprungen") + continue + + logger.info(f"\nPrüfe {modelName} ↔ {tableName}...") + + # Tabelle anpassen + if not _adaptTable(conn, tableName, modelFields, tableColumns, dryRun): + success = False + + # Spezialfall: UserInDB privilege → roleLabels + if modelName == "UserInDB": + if not _migratePrivilegeToRoleLabels(conn, tableName, dryRun): + success = False + + conn.close() + return success + + +def main(): + parser = argparse.ArgumentParser(description="Passt DB-Struktur an Pydantic-Modelle an") + parser.add_argument("--dry-run", action="store_true", help="Zeigt nur geplante Änderungen") + parser.add_argument("--db", help="Nur bestimmte DB(s) migrieren (komma-getrennt)") + args = parser.parse_args() + + databases = list(DATABASE_CONFIG.keys()) + if args.db: + databases = [db.strip() for db in args.db.split(",")] + + logger.info("="*60) + logger.info("DATENBANK-ANPASSUNG AN PYDANTIC-MODELLE") + logger.info("="*60) + logger.info(f"Modus: {'DRY-RUN' if args.dry_run else 'LIVE'}") + logger.info(f"Datenbanken: {', '.join(databases)}") + + if not args.dry_run: + response = input("\nÄnderungen durchführen? (yes/no): ") + if response.lower() != "yes": + logger.info("Abgebrochen") + return 0 + + allSuccess = True + for dbName in databases: + if not runMigration(dbName, args.dry_run): + allSuccess = False + + if allSuccess: + logger.info("\n" + "="*60) + logger.info("ALLE MIGRATIONEN ERFOLGREICH") + logger.info("="*60) + else: + logger.error("\n" + "="*60) + logger.error("EINIGE MIGRATIONEN HATTEN FEHLER") + logger.error("="*60) + + return 0 if allSuccess else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tool_db_export_migration.py b/tool_db_export_migration.py index 6c93370a..aea697c1 100644 --- a/tool_db_export_migration.py +++ b/tool_db_export_migration.py @@ -36,9 +36,36 @@ from datetime import datetime from typing import Dict, List, Any, Optional from pathlib import Path +# Add gateway to path for imports and set working directory +# Find gateway directory (could be in local/pending/ or gateway/) +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent +# If we're in local/pending/, go up to find gateway/ +if gatewayPath.name == "pending": + gatewayPath = gatewayPath.parent.parent / "gateway" +elif gatewayPath.name == "local": + gatewayPath = gatewayPath.parent / "gateway" +# If gateway doesn't exist, try current directory +if not gatewayPath.exists(): + gatewayPath = Path(__file__).parent.parent.parent / "gateway" +if gatewayPath.exists(): + sys.path.insert(0, str(gatewayPath)) + # Change working directory to gateway so APP_CONFIG can find .env file + os.chdir(str(gatewayPath)) +else: + # Fallback: assume we're already in gateway/ or add parent + sys.path.insert(0, str(Path(__file__).parent)) + # Try to change to gateway directory if it exists + potentialGateway = Path(__file__).parent + if potentialGateway.exists() and (potentialGateway / "modules" / "shared" / "configuration.py").exists(): + os.chdir(str(potentialGateway)) + import psycopg2 import psycopg2.extras +# Use the real APP_CONFIG which handles encryption and environment-specific configs +from modules.shared.configuration import APP_CONFIG # type: ignore + # Logging konfigurieren logging.basicConfig( level=logging.INFO, @@ -47,6 +74,27 @@ logging.basicConfig( ) logger = logging.getLogger(__name__) +# Force APP_CONFIG to refresh after changing working directory +# This ensures .env file is loaded from the correct location +try: + APP_CONFIG.refresh() + envType = APP_CONFIG.get('APP_ENV_TYPE', 'unknown') + keySysVar = APP_CONFIG.get('APP_KEY_SYSVAR', 'not_set') + logger.debug(f"APP_CONFIG refreshed. Environment type: {envType}") + logger.debug(f"APP_KEY_SYSVAR: {keySysVar}") + logger.debug(f"Current working directory: {os.getcwd()}") + + # Check if master key is available (needed for decrypting secrets) + if keySysVar != 'not_set': + masterKeyEnv = os.environ.get(keySysVar) + if masterKeyEnv: + logger.debug(f"Master key found in environment variable: {keySysVar}") + else: + logger.warning(f"Master key not found in environment variable: {keySysVar}") + logger.warning("Encrypted secrets may not be decryptable!") +except Exception as e: + logger.warning(f"Could not refresh APP_CONFIG: {e}") + # Alle PowerOn Datenbanken ALL_DATABASES = [ "poweron_app", # Haupt-App: User, Mandate, RBAC, Features @@ -56,65 +104,62 @@ ALL_DATABASES = [ "poweron_trustee", # Trustee ] - -def _loadEnvConfig() -> Dict[str, str]: - """Lädt die Konfiguration direkt aus der .env Datei.""" - config = {} - envPath = Path(__file__).parent / '.env' - - if not envPath.exists(): - logger.warning(f"Environment file not found at {envPath}") - return config - - # Versuche verschiedene Encodings - encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] - - for encoding in encodings: - try: - with open(envPath, 'r', encoding=encoding) as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' in line: - key, value = line.split('=', 1) - config[key.strip()] = value.strip() - # Erfolgreich geladen - return config - except UnicodeDecodeError: - continue - except Exception as e: - logger.error(f"Error loading .env file with {encoding}: {e}") - continue - - logger.error(f"Could not load .env file with any encoding") - return config - - -# Globale Konfiguration laden -_ENV_CONFIG = _loadEnvConfig() +# Datenbank-Konfiguration: Mapping von DB-Name zu Config-Prefix +# Jede Datenbank hat ihre eigenen Variablen: DB_APP_HOST, DB_CHAT_HOST, etc. +DATABASE_CONFIG = { + "poweron_app": "DB_APP", # DB_APP_HOST, DB_APP_USER, DB_APP_PASSWORD_SECRET, etc. + "poweron_chat": "DB_CHAT", # DB_CHAT_HOST, DB_CHAT_USER, etc. + "poweron_management": "DB_MANAGEMENT", + "poweron_realestate": "DB_REALESTATE", + "poweron_trustee": "DB_TRUSTEE", +} def _getConfigValue(key: str, default: str = None) -> str: - """Holt einen Konfigurationswert.""" - return _ENV_CONFIG.get(key, os.environ.get(key, default)) + """Holt einen Konfigurationswert über APP_CONFIG (unterstützt Verschlüsselung).""" + return APP_CONFIG.get(key, default) + + +def _getDbConfig(dbName: str) -> Dict[str, Any]: + """ + Holt die Datenbankverbindungs-Konfiguration für eine spezifische Datenbank. + Unterstützt sowohl DB-spezifische Variablen (DB_APP_HOST) als auch Fallback (DB_HOST). + """ + prefix = DATABASE_CONFIG.get(dbName, "DB") + + # Versuche zuerst DB-spezifische Variablen, dann Fallback auf allgemeine + host = _getConfigValue(f"{prefix}_HOST") or _getConfigValue("DB_HOST", "localhost") + user = _getConfigValue(f"{prefix}_USER") or _getConfigValue("DB_USER") + password = _getConfigValue(f"{prefix}_PASSWORD_SECRET") or _getConfigValue("DB_PASSWORD_SECRET") + port = _getConfigValue(f"{prefix}_PORT") or _getConfigValue("DB_PORT", "5432") + + return { + "host": host, + "user": user, + "password": password, + "port": int(port) if port else 5432 + } def _databaseExists(dbDatabase: str) -> bool: """Prüft ob eine Datenbank existiert.""" - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) + config = _getDbConfig(dbDatabase) + + if not config["user"]: + logger.warning(f"DB-User nicht gesetzt für Datenbank {dbDatabase}") + return False + if not config["password"]: + logger.warning(f"DB-Password nicht gesetzt für Datenbank {dbDatabase}") + return False try: # Verbinde zur postgres Datenbank um zu prüfen conn = psycopg2.connect( - host=dbHost, - port=dbPort, + host=config["host"], + port=config["port"], database="postgres", - user=dbUser, - password=dbPassword + user=config["user"], + password=config["password"] ) conn.autocommit = True @@ -128,37 +173,43 @@ def _databaseExists(dbDatabase: str) -> bool: conn.close() return exists except Exception as e: - logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") + logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase} auf {config['host']}:{config['port']}: {e}") return False def _getDbConnection(dbDatabase: str): """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" - # Erst prüfen ob Datenbank existiert - if not _databaseExists(dbDatabase): - logger.warning(f"Datenbank '{dbDatabase}' existiert nicht - übersprungen") + config = _getDbConfig(dbDatabase) + + # Prüfe ob wichtige Konfigurationswerte fehlen + if not config["user"]: + logger.error(f"DB-User nicht gesetzt für {dbDatabase} - kann keine Verbindung herstellen") + return None + if not config["password"]: + logger.error(f"DB-Password nicht gesetzt für {dbDatabase} - kann keine Verbindung herstellen") return None - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) + # Erst prüfen ob Datenbank existiert + if not _databaseExists(dbDatabase): + logger.warning(f"Datenbank '{dbDatabase}' existiert nicht auf {config['host']}:{config['port']} - übersprungen") + return None try: conn = psycopg2.connect( - host=dbHost, - port=dbPort, + host=config["host"], + port=config["port"], database=dbDatabase, - user=dbUser, - password=dbPassword, + user=config["user"], + password=config["password"], cursor_factory=psycopg2.extras.RealDictCursor ) # Autocommit muss VOR set_client_encoding gesetzt werden, um Transaction-Konflikte zu vermeiden conn.autocommit = True conn.set_client_encoding('UTF8') + logger.debug(f"Erfolgreich verbunden mit {config['host']}:{config['port']}/{dbDatabase}") return conn except Exception as e: - logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") + logger.error(f"Datenbankverbindung zu {dbDatabase} auf {config['host']}:{config['port']} fehlgeschlagen: {e}") raise @@ -504,6 +555,23 @@ def exportDatabase( # Welche Datenbanken exportieren? databasesToExport = onlyDatabases if onlyDatabases else ALL_DATABASES + # Prüfe ob grundlegende Konfigurationswerte vorhanden sind + # APP_CONFIG.get() entschlüsselt automatisch Werte, die mit _SECRET enden + # Jede Datenbank hat ihre eigenen Variablen (DB_APP_USER, DB_CHAT_USER, etc.) + missingConfigs = [] + for dbName in databasesToExport: + config = _getDbConfig(dbName) + if not config["user"] or not config["password"]: + missingConfigs.append(f"{dbName}: User={'gesetzt' if config['user'] else 'FEHLT'}, Password={'gesetzt' if config['password'] else 'FEHLT'}") + + if missingConfigs: + logger.error("WICHTIG: Einige Datenbank-Konfigurationen fehlen!") + for missing in missingConfigs: + logger.error(f" {missing}") + logger.error(" Bitte Umgebungsvariablen setzen oder .env Datei konfigurieren") + logger.error(" Hinweis: Verschlüsselte Secrets benötigen APP_KEY_SYSVAR Umgebungsvariable!") + logger.error(" Erwartete Variablen: DB_APP_USER, DB_CHAT_USER, DB_MANAGEMENT_USER, etc.") + # Standard-Ausgabepfad generieren (im Log-Ordner) if not outputPath: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") diff --git a/tool_db_import_migration.py b/tool_db_import_migration.py deleted file mode 100644 index 1ab4e4fe..00000000 --- a/tool_db_import_migration.py +++ /dev/null @@ -1,612 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Datenbank Import-Tool für Migration. - -Dieses Script importiert Daten aus einer JSON-Migrationsdatei -in ALLE PowerOn PostgreSQL-Datenbanken. - -ACHTUNG: Dieses Script kann bestehende Daten überschreiben! -Bitte vor dem Import ein Backup erstellen. - -Datenbanken: - - poweron_app (User, Mandate, RBAC, Features, etc.) - - poweron_chat (Chat-Konversationen und Nachrichten) - - poweron_management (Workflows, Prompts, Connections, etc.) - - poweron_realestate (Real Estate Daten) - - poweron_trustee (Trustee Daten) - -Verwendung: - python tool_db_import_migration.py [--dry-run] [--force] - -Optionen: - --dry-run Simuliert den Import ohne Änderungen - --force Bestätigung überspringen - --clear-first Tabellen vor dem Import leeren - --only-tables Komma-getrennte Liste von Tabellen (nur diese importieren) - --only-db Komma-getrennte Liste von Datenbanken (nur diese importieren) -""" - -import os -import sys -import json -import argparse -import logging -import time -from datetime import datetime -from typing import Dict, List, Any, Optional -from pathlib import Path - -import psycopg2 -import psycopg2.extras - -# Logging konfigurieren -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%Y-%m-%d %H:%M:%S' -) -logger = logging.getLogger(__name__) - -# Alle PowerOn Datenbanken -ALL_DATABASES = [ - "poweron_app", - "poweron_chat", - "poweron_management", - "poweron_realestate", - "poweron_trustee", -] - - -def _loadEnvConfig() -> Dict[str, str]: - """Lädt die Konfiguration direkt aus der .env Datei.""" - config = {} - envPath = Path(__file__).parent / '.env' - - if not envPath.exists(): - logger.warning(f"Environment file not found at {envPath}") - return config - - # Versuche verschiedene Encodings - encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252'] - - for encoding in encodings: - try: - with open(envPath, 'r', encoding=encoding) as f: - for line in f: - line = line.strip() - if not line or line.startswith('#'): - continue - if '=' in line: - key, value = line.split('=', 1) - config[key.strip()] = value.strip() - # Erfolgreich geladen - return config - except UnicodeDecodeError: - continue - except Exception as e: - logger.error(f"Error loading .env file with {encoding}: {e}") - continue - - logger.error(f"Could not load .env file with any encoding") - return config - - -# Globale Konfiguration laden -_ENV_CONFIG = _loadEnvConfig() - - -def _getConfigValue(key: str, default: str = None) -> str: - """Holt einen Konfigurationswert.""" - return _ENV_CONFIG.get(key, os.environ.get(key, default)) - - -def _getUtcTimestamp() -> float: - """Gibt den aktuellen UTC-Timestamp zurück.""" - return time.time() - - -def _databaseExists(dbDatabase: str) -> bool: - """Prüft ob eine Datenbank existiert.""" - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) - - try: - conn = psycopg2.connect( - host=dbHost, - port=dbPort, - database="postgres", - user=dbUser, - password=dbPassword - ) - conn.autocommit = True - - with conn.cursor() as cursor: - cursor.execute( - "SELECT 1 FROM pg_database WHERE datname = %s", - (dbDatabase,) - ) - exists = cursor.fetchone() is not None - - conn.close() - return exists - except Exception as e: - logger.error(f"Fehler beim Prüfen der Datenbank {dbDatabase}: {e}") - return False - - -def _getDbConnection(dbDatabase: str, autocommit: bool = False): - """Erstellt eine Verbindung zu einer spezifischen PostgreSQL-Datenbank.""" - # Erst prüfen ob Datenbank existiert - if not _databaseExists(dbDatabase): - logger.warning(f"Datenbank '{dbDatabase}' existiert nicht") - return None - - dbHost = _getConfigValue("DB_HOST", "localhost") - dbUser = _getConfigValue("DB_USER") - dbPassword = _getConfigValue("DB_PASSWORD_SECRET") - dbPort = int(_getConfigValue("DB_PORT", "5432")) - - try: - conn = psycopg2.connect( - host=dbHost, - port=dbPort, - database=dbDatabase, - user=dbUser, - password=dbPassword, - cursor_factory=psycopg2.extras.RealDictCursor - ) - conn.set_client_encoding('UTF8') - conn.autocommit = autocommit - return conn - except Exception as e: - logger.error(f"Datenbankverbindung zu {dbDatabase} fehlgeschlagen: {e}") - raise - - -def _getExistingTables(conn) -> List[str]: - """Gibt alle Tabellennamen in der Datenbank zurück.""" - with conn.cursor() as cursor: - cursor.execute(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_type = 'BASE TABLE' - ORDER BY table_name - """) - tables = [row["table_name"] for row in cursor.fetchall()] - return tables - - -def _getTableColumns(conn, tableName: str) -> List[str]: - """Gibt alle Spalten einer Tabelle zurück.""" - with conn.cursor() as cursor: - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = %s AND table_schema = 'public' - """, (tableName,)) - columns = [row["column_name"] for row in cursor.fetchall()] - return columns - - -def _clearTable(conn, tableName: str): - """Löscht alle Daten aus einer Tabelle.""" - with conn.cursor() as cursor: - cursor.execute(f'DELETE FROM "{tableName}"') - - -def _insertRecord(conn, tableName: str, record: Dict[str, Any], existingColumns: List[str]) -> bool: - """Fügt einen Datensatz in eine Tabelle ein (UPSERT).""" - filteredRecord = {k: v for k, v in record.items() if k in existingColumns} - - if not filteredRecord: - return False - - # Metadaten hinzufügen falls nicht vorhanden - currentTime = _getUtcTimestamp() - if "_createdAt" not in filteredRecord and "_createdAt" in existingColumns: - filteredRecord["_createdAt"] = currentTime - if "_modifiedAt" in existingColumns: - filteredRecord["_modifiedAt"] = currentTime - - columns = list(filteredRecord.keys()) - values = [] - - for col in columns: - value = filteredRecord[col] - if isinstance(value, (dict, list)): - values.append(json.dumps(value)) - else: - values.append(value) - - colNames = ", ".join([f'"{col}"' for col in columns]) - placeholders = ", ".join(["%s"] * len(columns)) - - updateCols = [col for col in columns if col not in ["id", "_createdAt", "_createdBy"]] - updateClause = ", ".join([f'"{col}" = EXCLUDED."{col}"' for col in updateCols]) - - if updateClause: - sql = f''' - INSERT INTO "{tableName}" ({colNames}) - VALUES ({placeholders}) - ON CONFLICT ("id") DO UPDATE SET {updateClause} - ''' - else: - sql = f''' - INSERT INTO "{tableName}" ({colNames}) - VALUES ({placeholders}) - ON CONFLICT ("id") DO NOTHING - ''' - - try: - with conn.cursor() as cursor: - cursor.execute(sql, values) - return True - except Exception as e: - logger.error(f"Fehler beim Einfügen in {tableName}: {e}") - return False - - -def loadMigrationFile(filePath: str) -> Dict[str, Any]: - """Lädt die Migrationsdatei.""" - logger.info(f"Lade Migrationsdatei: {filePath}") - - if not os.path.exists(filePath): - raise FileNotFoundError(f"Datei nicht gefunden: {filePath}") - - with open(filePath, "r", encoding="utf-8") as f: - data = json.load(f) - - # Validierung - unterstütze beide Formate (alt: tables, neu: databases) - if "databases" not in data and "tables" not in data: - raise ValueError("Ungültiges Migrationsformat: 'databases' oder 'tables' erforderlich") - - return data - - -def _importSingleDatabase( - dbName: str, - dbData: Dict[str, Any], - dryRun: bool, - clearFirst: bool, - onlyTables: Optional[List[str]] -) -> Dict[str, Any]: - """Importiert Daten in eine einzelne Datenbank.""" - stats = { - "imported": {}, - "skipped": {}, - "errors": {}, - "totalImported": 0, - "totalSkipped": 0, - "totalErrors": 0 - } - - conn = _getDbConnection(dbName) - if conn is None: - logger.warning(f" Datenbank '{dbName}' existiert nicht - übersprungen") - return stats - - try: - existingTables = _getExistingTables(conn) - tables = dbData.get("tables", {}) - - tablesToImport = list(tables.keys()) - if onlyTables: - tablesToImport = [t for t in tablesToImport if t in onlyTables] - - for tableName in tablesToImport: - records = tables[tableName] - - if tableName not in existingTables: - logger.warning(f" Tabelle '{tableName}' existiert nicht - übersprungen") - stats["skipped"][tableName] = len(records) - stats["totalSkipped"] += len(records) - continue - - if dryRun: - stats["imported"][tableName] = len(records) - stats["totalImported"] += len(records) - continue - - if clearFirst: - _clearTable(conn, tableName) - - existingColumns = _getTableColumns(conn, tableName) - - imported = 0 - errors = 0 - - for record in records: - if _insertRecord(conn, tableName, record, existingColumns): - imported += 1 - else: - errors += 1 - - stats["imported"][tableName] = imported - stats["totalImported"] += imported - - if errors > 0: - stats["errors"][tableName] = errors - stats["totalErrors"] += errors - - if imported > 0: - logger.info(f" {tableName}: {imported} importiert, {errors} Fehler") - - if not dryRun: - conn.commit() - else: - conn.rollback() - - return stats - - except Exception as e: - conn.rollback() - logger.error(f" Import fehlgeschlagen: {e}") - raise - - finally: - conn.close() - - -def importDatabase( - filePath: str, - dryRun: bool = False, - clearFirst: bool = False, - onlyTables: Optional[List[str]] = None, - onlyDatabases: Optional[List[str]] = None -) -> Dict[str, Any]: - """ - Importiert Daten aus einer Migrationsdatei. - - Args: - filePath: Pfad zur Migrationsdatei - dryRun: Nur simulieren - clearFirst: Tabellen vor Import leeren - onlyTables: Nur diese Tabellen importieren - onlyDatabases: Nur diese Datenbanken importieren - - Returns: - Import-Statistiken - """ - migrationData = loadMigrationFile(filePath) - meta = migrationData.get("meta", {}) - - logger.info(f"Migrationsdatei geladen:") - logger.info(f" Exportiert am: {meta.get('exportedAt', 'unbekannt')}") - logger.info(f" Quelle: {meta.get('exportedFrom', 'unbekannt')}") - - stats = { - "databases": {}, - "totalImported": 0, - "totalSkipped": 0, - "totalErrors": 0 - } - - # Neues Format (mehrere Datenbanken) - if "databases" in migrationData: - databases = migrationData["databases"] - logger.info(f" Datenbanken: {len(databases)}") - logger.info(f" Tabellen: {meta.get('totalTables', 'unbekannt')}") - logger.info(f" Datensätze: {meta.get('totalRecords', 'unbekannt')}") - - for dbName, dbData in databases.items(): - if onlyDatabases and dbName not in onlyDatabases: - continue - - logger.info(f"Importiere Datenbank: {dbName}") - dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) - - stats["databases"][dbName] = dbStats - stats["totalImported"] += dbStats["totalImported"] - stats["totalSkipped"] += dbStats["totalSkipped"] - stats["totalErrors"] += dbStats["totalErrors"] - - # Altes Format (einzelne Datenbank - poweron_app) - elif "tables" in migrationData: - logger.info(" Format: Legacy (einzelne Datenbank)") - dbName = "poweron_app" - dbData = {"tables": migrationData["tables"]} - - if not onlyDatabases or dbName in onlyDatabases: - logger.info(f"Importiere Datenbank: {dbName}") - dbStats = _importSingleDatabase(dbName, dbData, dryRun, clearFirst, onlyTables) - - stats["databases"][dbName] = dbStats - stats["totalImported"] = dbStats["totalImported"] - stats["totalSkipped"] = dbStats["totalSkipped"] - stats["totalErrors"] = dbStats["totalErrors"] - - if dryRun: - logger.info("Dry-Run: Keine Änderungen vorgenommen") - - return stats - - -def printImportPreview(filePath: str): - """Zeigt eine Vorschau der zu importierenden Daten.""" - migrationData = loadMigrationFile(filePath) - meta = migrationData.get("meta", {}) - - print("\n" + "=" * 70) - print("IMPORT VORSCHAU") - print("=" * 70) - print(f"Datei: {filePath}") - print(f"Exportiert am: {meta.get('exportedAt', 'unbekannt')}") - print(f"Quelle: {meta.get('exportedFrom', 'unbekannt')}") - - # Neues Format - if "databases" in migrationData: - databases = migrationData["databases"] - print(f"Datenbanken: {len(databases)}") - print("=" * 70) - - grandTotal = 0 - for dbName, dbData in databases.items(): - tables = dbData.get("tables", {}) - dbTotal = sum(len(records) for records in tables.values()) - grandTotal += dbTotal - - print(f"\n{dbName} ({dbTotal} Datensätze)") - print("-" * 70) - print(f" {'Tabelle':<45} {'Datensätze':>15}") - print(f" {'-' * 45} {'-' * 15}") - - for tableName, records in sorted(tables.items()): - if len(records) > 0: - print(f" {tableName:<45} {len(records):>15}") - - print("\n" + "=" * 70) - print(f"GESAMT: {grandTotal} Datensätze") - - # Altes Format - elif "tables" in migrationData: - tables = migrationData["tables"] - print(f"Format: Legacy (poweron_app)") - print("-" * 70) - print(f"{'Tabelle':<45} {'Datensätze':>15}") - print("-" * 70) - - totalRecords = 0 - for tableName, records in sorted(tables.items()): - count = len(records) - totalRecords += count - if count > 0: - print(f"{tableName:<45} {count:>15}") - - print("-" * 70) - print(f"{'GESAMT':<45} {totalRecords:>15}") - - print("=" * 70 + "\n") - - -def main(): - parser = argparse.ArgumentParser( - description="Importiert Datenbank-Daten aus einer Migrationsdatei", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Datenbanken: - poweron_app - User, Mandate, RBAC, Features - poweron_chat - Chat-Konversationen - poweron_management - Workflows, Prompts, Connections - poweron_realestate - Real Estate Daten - poweron_trustee - Trustee Daten - -Beispiele: - python tool_db_import_migration.py migration_export.json --dry-run - python tool_db_import_migration.py migration_export.json --preview - python tool_db_import_migration.py migration_export.json --force - python tool_db_import_migration.py migration_export.json --clear-first --force - python tool_db_import_migration.py migration_export.json --only-db poweron_app - python tool_db_import_migration.py migration_export.json --only-tables UserInDB,Mandate - """ - ) - - parser.add_argument( - "import_file", - help="Pfad zur Migrationsdatei (JSON)", - type=str - ) - - parser.add_argument( - "--dry-run", - help="Simuliert den Import ohne Änderungen", - action="store_true" - ) - - parser.add_argument( - "--force", - help="Bestätigung überspringen", - action="store_true" - ) - - parser.add_argument( - "--clear-first", - help="Tabellen vor dem Import leeren", - action="store_true" - ) - - parser.add_argument( - "--only-tables", - help="Nur diese Tabellen importieren (komma-getrennt)", - type=str, - default="" - ) - - parser.add_argument( - "--only-db", - help="Nur diese Datenbank(en) importieren (komma-getrennt)", - type=str, - default="" - ) - - parser.add_argument( - "--preview", - help="Nur Vorschau anzeigen (kein Import)", - action="store_true" - ) - - args = parser.parse_args() - - # Nur Vorschau anzeigen - if args.preview: - printImportPreview(args.import_file) - return - - # Listen parsen - onlyTables = None - if args.only_tables: - onlyTables = [t.strip() for t in args.only_tables.split(",") if t.strip()] - - onlyDatabases = None - if args.only_db: - onlyDatabases = [db.strip() for db in args.only_db.split(",") if db.strip()] - - # Bestätigung einholen - if not args.dry_run and not args.force: - printImportPreview(args.import_file) - - if args.clear_first: - print("WARNUNG: --clear-first wird ALLE bestehenden Daten in den Zieltabellen löschen!") - - response = input("\nMöchten Sie den Import starten? [y/N]: ") - if response.lower() not in ["y", "yes", "j", "ja"]: - print("Import abgebrochen.") - return - - # Import durchführen - try: - if args.dry_run: - logger.info("=== DRY-RUN MODUS ===") - - stats = importDatabase( - filePath=args.import_file, - dryRun=args.dry_run, - clearFirst=args.clear_first, - onlyTables=onlyTables, - onlyDatabases=onlyDatabases - ) - - print("\n" + "=" * 70) - print("IMPORT ERGEBNIS") - print("=" * 70) - print(f"Importiert: {stats['totalImported']} Datensätze") - print(f"Übersprungen: {stats['totalSkipped']} Datensätze") - print(f"Fehler: {stats['totalErrors']} Datensätze") - - if args.dry_run: - print("\n(Dry-Run: Keine tatsächlichen Änderungen vorgenommen)") - else: - print("\n Import erfolgreich abgeschlossen!") - - print("=" * 70 + "\n") - - except Exception as e: - logger.error(f"Import fehlgeschlagen: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main()