From f3b01c823e25728b3f53390c52798a810a279a69 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 21 Jan 2026 10:34:42 +0100 Subject: [PATCH] saas multi-mandate rbac up and running --- modules/connectors/connectorDbPostgre.py | 8 +- modules/interfaces/interfaceBootstrap.py | 170 +++++++ modules/routes/routeFeatures.py | 463 ++++++++++++++++++ modules/routes/routeRbac.py | 20 +- .../script_db_adapt_to_models.py | 4 +- scripts/script_db_cleanup_duplicate_roles.py | 189 +++++++ .../script_db_export_migration.py | 27 +- .../script_security_encrypt_all_env_files.py | 25 +- .../script_security_encrypt_config_value.py | 21 +- .../script_security_generate_master_keys.py | 12 +- .../script_stats_durations_from_log.py | 0 .../script_stats_get_codelines.py | 0 .../script_stats_showUnusedFunctions.py | 11 +- 13 files changed, 903 insertions(+), 47 deletions(-) rename tool_db_adapt_to_models.py => scripts/script_db_adapt_to_models.py (99%) create mode 100644 scripts/script_db_cleanup_duplicate_roles.py rename tool_db_export_migration.py => scripts/script_db_export_migration.py (97%) rename tool_security_encrypt_all_env_files.py => scripts/script_security_encrypt_all_env_files.py (95%) rename tool_security_encrypt_config_value.py => scripts/script_security_encrypt_config_value.py (96%) rename tool_security_generate_master_keys.py => scripts/script_security_generate_master_keys.py (88%) rename tool_stats_durations_from_log.py => scripts/script_stats_durations_from_log.py (100%) rename tool_stats_get_codelines.py => scripts/script_stats_get_codelines.py (100%) rename tool_stats_showUnusedFunctions.py => scripts/script_stats_showUnusedFunctions.py (96%) diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 5cf2dc62..2dfec2b4 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -853,8 +853,12 @@ class DatabaseConnector: if recordFilter: for field, value in recordFilter.items(): - where_conditions.append(f'"{field}" = %s') - where_values.append(value) + if value is None: + # Use IS NULL for None values (= NULL is always false in SQL) + where_conditions.append(f'"{field}" IS NULL') + else: + where_conditions.append(f'"{field}" = %s') + where_values.append(value) # Build the query if where_conditions: diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 1d5f1139..545b0040 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -298,9 +298,179 @@ def initFeatures(db: DatabaseConnector) -> None: else: logger.debug(f"Feature {feature.code} already exists") + # Initialize feature-specific template roles + _initFeatureTemplateRoles(db) + logger.info("Features initialization completed") +def _initFeatureTemplateRoles(db: DatabaseConnector) -> None: + """ + Initialize feature-specific template roles. + + These are global template roles (mandateId=None, featureInstanceId=None) + that get copied when a new FeatureInstance is created. + + Template roles are NOT system roles (isSystemRole=False) and can be + modified or deleted by administrators. + + Args: + db: Database connector instance + """ + logger.info("Initializing feature-specific template roles") + + # Feature-specific template roles definition + # Each feature has its own set of roles with appropriate descriptions + featureTemplateRoles = { + "trustee": [ + { + "roleLabel": "trustee-admin", + "description": { + "en": "Trustee Administrator - Full access to all trustee data and settings", + "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", + "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" + } + }, + { + "roleLabel": "trustee-accountant", + "description": { + "en": "Trustee Accountant - Manage accounting and financial data", + "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", + "fr": "Comptable fiduciaire - Gérer les données comptables et financières" + } + }, + { + "roleLabel": "trustee-client", + "description": { + "en": "Trustee Client - View own accounting data and documents", + "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", + "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" + } + }, + ], + "chatbot": [ + { + "roleLabel": "chatbot-admin", + "description": { + "en": "Chatbot Administrator - Full access to chatbot settings and all conversations", + "de": "Chatbot-Administrator - Vollzugriff auf Chatbot-Einstellungen und alle Konversationen", + "fr": "Administrateur chatbot - Accès complet aux paramètres et conversations" + } + }, + { + "roleLabel": "chatbot-user", + "description": { + "en": "Chatbot User - Use chatbot and view own conversations", + "de": "Chatbot-Benutzer - Chatbot nutzen und eigene Konversationen einsehen", + "fr": "Utilisateur chatbot - Utiliser le chatbot et consulter ses conversations" + } + }, + ], + "chatworkflow": [ + { + "roleLabel": "workflow-admin", + "description": { + "en": "Workflow Administrator - Full access to workflow configuration and execution", + "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", + "fr": "Administrateur workflow - Accès complet à la configuration et exécution" + } + }, + { + "roleLabel": "workflow-editor", + "description": { + "en": "Workflow Editor - Create and modify workflows", + "de": "Workflow-Editor - Workflows erstellen und bearbeiten", + "fr": "Éditeur workflow - Créer et modifier les workflows" + } + }, + { + "roleLabel": "workflow-viewer", + "description": { + "en": "Workflow Viewer - View workflows and execution results", + "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", + "fr": "Visualiseur workflow - Consulter les workflows et résultats" + } + }, + ], + "neutralization": [ + { + "roleLabel": "neutralization-admin", + "description": { + "en": "Neutralization Administrator - Full access to neutralization settings and data", + "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", + "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" + } + }, + { + "roleLabel": "neutralization-analyst", + "description": { + "en": "Neutralization Analyst - Analyze and process neutralization data", + "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", + "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" + } + }, + ], + "realestate": [ + { + "roleLabel": "realestate-admin", + "description": { + "en": "Real Estate Administrator - Full access to all property data and settings", + "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", + "fr": "Administrateur immobilier - Accès complet aux données et paramètres" + } + }, + { + "roleLabel": "realestate-manager", + "description": { + "en": "Real Estate Manager - Manage properties and tenants", + "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", + "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" + } + }, + { + "roleLabel": "realestate-viewer", + "description": { + "en": "Real Estate Viewer - View property information", + "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", + "fr": "Visualiseur immobilier - Consulter les informations immobilières" + } + }, + ], + } + + # Get existing template roles (mandateId=None, featureCode set) + existingRoles = db.getRecordset(Role, recordFilter={"mandateId": None}) + existingRoleKeys = { + (r.get("featureCode"), r.get("roleLabel")) + for r in existingRoles + if r.get("featureCode") is not None + } + + createdCount = 0 + for featureCode, roles in featureTemplateRoles.items(): + for roleDef in roles: + roleKey = (featureCode, roleDef["roleLabel"]) + if roleKey not in existingRoleKeys: + try: + templateRole = Role( + roleLabel=roleDef["roleLabel"], + description=roleDef["description"], + mandateId=None, # Global template role + featureInstanceId=None, + featureCode=featureCode, + isSystemRole=False # Can be deleted by admins + ) + db.recordCreate(Role, templateRole) + createdCount += 1 + logger.info(f"Created template role: {roleDef['roleLabel']} for feature {featureCode}") + except Exception as e: + logger.warning(f"Error creating template role {roleDef['roleLabel']}: {e}") + else: + logger.debug(f"Template role {roleDef['roleLabel']} for {featureCode} already exists") + + logger.info(f"Feature template roles initialization completed ({createdCount} created)") + + def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]: """ Get role ID by label, using cache or database lookup. diff --git a/modules/routes/routeFeatures.py b/modules/routes/routeFeatures.py index 326adeb7..a9a124a8 100644 --- a/modules/routes/routeFeatures.py +++ b/modules/routes/routeFeatures.py @@ -709,6 +709,469 @@ async def createTemplateRole( ) +# ============================================================================= +# Feature Instance Users Endpoints +# Manage which users have access to a specific feature instance +# ============================================================================= + +class FeatureInstanceUserCreate(BaseModel): + """Request model for adding a user to a feature instance""" + userId: str = Field(..., description="User ID to add") + roleIds: List[str] = Field(default_factory=list, description="Role IDs to assign") + + +class FeatureInstanceUserResponse(BaseModel): + """Response model for a user in a feature instance""" + userId: str + username: str + email: Optional[str] + fullName: Optional[str] + featureAccessId: str + roleIds: List[str] + roleLabels: List[str] + enabled: bool + + +@router.get("/instances/{instanceId}/users", response_model=List[FeatureInstanceUserResponse]) +@limiter.limit("60/minute") +async def listFeatureInstanceUsers( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> List[FeatureInstanceUserResponse]: + """ + List all users with access to a specific feature instance. + + Returns users and their roles for the given instance. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access (unless SysAdmin) + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Get all FeatureAccess records for this instance + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + from modules.datamodels.datamodelRbac import Role + from modules.datamodels.datamodelUam import UserInDB + + featureAccesses = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"featureInstanceId": instanceId} + ) + + result = [] + for fa in featureAccesses: + userId = fa.get("userId") + featureAccessId = fa.get("id") + + # Get user info + users = rootInterface.db.getRecordset(UserInDB, recordFilter={"id": userId}) + if not users: + continue + user = users[0] + + # Get role IDs via FeatureAccessRole junction table + featureAccessRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + roleIds = [far.get("roleId") for far in featureAccessRoles] + + # Get role labels + roleLabels = [] + for roleId in roleIds: + roles = rootInterface.db.getRecordset(Role, recordFilter={"id": roleId}) + if roles: + roleLabels.append(roles[0].get("roleLabel", "")) + + result.append(FeatureInstanceUserResponse( + userId=userId, + username=user.get("username", ""), + email=user.get("email"), + fullName=user.get("fullName"), + featureAccessId=featureAccessId, + roleIds=roleIds, + roleLabels=roleLabels, + enabled=fa.get("enabled", True) + )) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error listing feature instance users: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to list feature instance users: {str(e)}" + ) + + +@router.post("/instances/{instanceId}/users", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def addUserToFeatureInstance( + request: Request, + instanceId: str, + data: FeatureInstanceUserCreate, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Add a user to a feature instance with specified roles. + + Creates a FeatureAccess record and associated FeatureAccessRole records. + + Args: + instanceId: FeatureInstance ID + data: User and role data + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to add users to feature instances" + ) + + # Verify user exists + from modules.datamodels.datamodelUam import UserInDB + users = rootInterface.db.getRecordset(UserInDB, recordFilter={"id": data.userId}) + if not users: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"User '{data.userId}' not found" + ) + + # Check if user already has access + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": data.userId, "featureInstanceId": instanceId} + ) + if existingAccess: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="User already has access to this feature instance" + ) + + # Create FeatureAccess record + featureAccess = FeatureAccess( + userId=data.userId, + featureInstanceId=instanceId, + enabled=True + ) + createdAccess = rootInterface.db.recordCreate(FeatureAccess, featureAccess.model_dump()) + featureAccessId = createdAccess.get("id") + + # Create FeatureAccessRole records for each role + for roleId in data.roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + + logger.info( + f"User {context.user.id} added user {data.userId} to feature instance {instanceId} " + f"with roles {data.roleIds}" + ) + + return { + "featureAccessId": featureAccessId, + "userId": data.userId, + "featureInstanceId": instanceId, + "roleIds": data.roleIds, + "enabled": True + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error adding user to feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to add user to feature instance: {str(e)}" + ) + + +@router.delete("/instances/{instanceId}/users/{userId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +async def removeUserFromFeatureInstance( + request: Request, + instanceId: str, + userId: str, + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, str]: + """ + Remove a user's access from a feature instance. + + Deletes the FeatureAccess record (CASCADE will delete FeatureAccessRole records). + + Args: + instanceId: FeatureInstance ID + userId: User ID to remove + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to remove users from feature instances" + ) + + # Find FeatureAccess record + from modules.datamodels.datamodelMembership import FeatureAccess + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) + if not existingAccess: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User does not have access to this feature instance" + ) + + featureAccessId = existingAccess[0].get("id") + + # Delete FeatureAccess (CASCADE will delete FeatureAccessRole records) + rootInterface.db.recordDelete(FeatureAccess, featureAccessId) + + logger.info( + f"User {context.user.id} removed user {userId} from feature instance {instanceId}" + ) + + return { + "message": "User access removed", + "userId": userId, + "featureInstanceId": instanceId + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error removing user from feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to remove user from feature instance: {str(e)}" + ) + + +@router.put("/instances/{instanceId}/users/{userId}/roles", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +async def updateFeatureInstanceUserRoles( + request: Request, + instanceId: str, + userId: str, + roleIds: List[str], + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Update a user's roles in a feature instance. + + Replaces all existing FeatureAccessRole records with new ones. + + Args: + instanceId: FeatureInstance ID + userId: User ID to update + roleIds: New list of role IDs + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Check admin permission + if not _hasMandateAdminRole(context) and not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required to update user roles" + ) + + # Find FeatureAccess record + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + existingAccess = rootInterface.db.getRecordset( + FeatureAccess, + recordFilter={"userId": userId, "featureInstanceId": instanceId} + ) + if not existingAccess: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User does not have access to this feature instance" + ) + + featureAccessId = existingAccess[0].get("id") + + # Delete existing FeatureAccessRole records + existingRoles = rootInterface.db.getRecordset( + FeatureAccessRole, + recordFilter={"featureAccessId": featureAccessId} + ) + for role in existingRoles: + rootInterface.db.recordDelete(FeatureAccessRole, role.get("id")) + + # Create new FeatureAccessRole records + for roleId in roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + + logger.info( + f"User {context.user.id} updated roles for user {userId} in feature instance {instanceId}: {roleIds}" + ) + + return { + "featureAccessId": featureAccessId, + "userId": userId, + "featureInstanceId": instanceId, + "roleIds": roleIds + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating user roles in feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to update user roles: {str(e)}" + ) + + +@router.get("/instances/{instanceId}/available-roles", response_model=List[Dict[str, Any]]) +@limiter.limit("60/minute") +async def getFeatureInstanceAvailableRoles( + request: Request, + instanceId: str, + context: RequestContext = Depends(getRequestContext) +) -> List[Dict[str, Any]]: + """ + Get available roles for a feature instance. + + Returns instance-specific roles (copied from templates) that can be assigned to users. + + Args: + instanceId: FeatureInstance ID + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + # Verify instance exists + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Feature instance '{instanceId}' not found" + ) + + # Verify mandate access + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.isSysAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Access denied to this feature instance" + ) + + # Get roles for this instance + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset( + Role, + recordFilter={"featureInstanceId": instanceId} + ) + + result = [] + for role in instanceRoles: + result.append({ + "id": role.get("id"), + "roleLabel": role.get("roleLabel"), + "description": role.get("description", {}), + "featureCode": role.get("featureCode"), + "isSystemRole": role.get("isSystemRole", False) + }) + + return result + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting available roles for feature instance: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get available roles: {str(e)}" + ) + + # ============================================================================= # Dynamic Feature Route (MUST be last to avoid catching /instances, /my, etc.) # ============================================================================= diff --git a/modules/routes/routeRbac.py b/modules/routes/routeRbac.py index 03c66ae0..5592bfa1 100644 --- a/modules/routes/routeRbac.py +++ b/modules/routes/routeRbac.py @@ -565,12 +565,20 @@ async def deleteAccessRule( async def listRoles( request: Request, pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), + includeTemplates: bool = Query(False, description="Include feature template roles"), currentUser: User = Depends(requireSysAdmin) ) -> PaginatedResponse: """ - Get list of all available roles with metadata. + Get list of global/system roles with metadata. MULTI-TENANT: SysAdmin-only (roles are system resources). + By default, only returns true global roles (mandateId=None, featureInstanceId=None, featureCode=None). + Feature template roles are managed via /api/features/templates/roles. + + Args: + pagination: Optional pagination parameters + includeTemplates: If True, also include feature template roles (featureCode != None) + Returns: - List of role dictionaries with role label, description, and user count """ @@ -598,8 +606,18 @@ async def listRoles( roleCounts = interface.countRoleAssignments() # Convert Role objects to dictionaries and add user counts + # Filter to only global roles (mandateId=None, featureInstanceId=None) + # Unless includeTemplates=True, also exclude feature template roles (featureCode != None) result = [] for role in dbRoles: + # Filter: Only global roles (no mandate, no instance) + if role.mandateId is not None or role.featureInstanceId is not None: + continue + + # Filter: Exclude feature template roles unless includeTemplates=True + if not includeTemplates and role.featureCode is not None: + continue + result.append({ "id": role.id, "roleLabel": role.roleLabel, diff --git a/tool_db_adapt_to_models.py b/scripts/script_db_adapt_to_models.py similarity index 99% rename from tool_db_adapt_to_models.py rename to scripts/script_db_adapt_to_models.py index 85c6e8fc..163c4cb8 100644 --- a/tool_db_adapt_to_models.py +++ b/scripts/script_db_adapt_to_models.py @@ -8,7 +8,7 @@ Einfaches Script das: 3. Spezialfall: UserInDB.privilege → roleLabels migriert Verwendung: - python tool_db_adapt_to_models.py [--dry-run] [--db ] + python script_db_adapt_to_models.py [--dry-run] [--db ] """ import os @@ -21,7 +21,7 @@ from typing import Dict, List, Any, Optional # Gateway-Pfad setzen scriptPath = Path(__file__).resolve() -gatewayPath = scriptPath.parent +gatewayPath = scriptPath.parent.parent sys.path.insert(0, str(gatewayPath)) os.chdir(str(gatewayPath)) diff --git a/scripts/script_db_cleanup_duplicate_roles.py b/scripts/script_db_cleanup_duplicate_roles.py new file mode 100644 index 00000000..392cde80 --- /dev/null +++ b/scripts/script_db_cleanup_duplicate_roles.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +""" +Cleanup script for duplicate roles in the database. + +This script removes duplicate Role records that were created due to the IS NULL bug +in connectorDbPostgre.py. The bug caused `mandateId = NULL` to always return FALSE, +which meant the duplicate check in bootstrap didn't work. + +Usage: + python cleanupDuplicateRoles.py + +The script will: +1. Find all duplicate roles (same roleLabel + featureCode + featureInstanceId + mandateId) +2. Keep the oldest one (first created) and delete the rest +3. Report the number of deleted roles +""" + +import sys +import os + +# Add parent directory to path +gatewayDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, gatewayDir) + +# Load environment variables from env_dev.env +from dotenv import load_dotenv +envPath = os.path.join(gatewayDir, "env_dev.env") +if os.path.exists(envPath): + load_dotenv(envPath) + +from modules.datamodels.datamodelRbac import Role +from modules.security.rootAccess import getRootDbAppConnector +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def _getDbConnector(): + """Get a database connector using the application's configuration.""" + return getRootDbAppConnector() + + +def cleanupDuplicateRoles(): + """ + Clean up duplicate roles in the database. + + Keeps the first role (by ID, which is UUID-based) for each unique combination of: + - roleLabel + - featureCode + - featureInstanceId + - mandateId + """ + db = _getDbConnector() + + # Get all roles + allRoles = db.getRecordset(Role, recordFilter=None) + logger.info(f"Found {len(allRoles)} total roles in database") + + # Group roles by their unique key + roleGroups = {} + for role in allRoles: + # Create a key tuple for grouping + # Note: None values need special handling for dict keys + key = ( + role.get("roleLabel"), + role.get("featureCode") or "__NONE__", + role.get("featureInstanceId") or "__NONE__", + role.get("mandateId") or "__NONE__" + ) + + if key not in roleGroups: + roleGroups[key] = [] + roleGroups[key].append(role) + + # Find and delete duplicates + deletedCount = 0 + for key, roles in roleGroups.items(): + if len(roles) > 1: + # Sort by ID (UUID, string comparison works for finding "first") + # Actually, we want to keep one - let's keep by created order if available + # Since there's no createdAt, we'll just keep the first one + toKeep = roles[0] + toDelete = roles[1:] + + logger.info(f"Found {len(roles)} duplicates for key {key}") + logger.info(f" Keeping: {toKeep.get('id')} ({toKeep.get('roleLabel')})") + + for role in toDelete: + roleId = role.get("id") + try: + db.recordDelete(Role, roleId) + deletedCount += 1 + logger.info(f" Deleted: {roleId}") + except Exception as e: + logger.error(f" Failed to delete {roleId}: {e}") + + logger.info(f"Cleanup complete: {deletedCount} duplicate roles deleted") + logger.info(f"Remaining roles: {len(allRoles) - deletedCount}") + + return deletedCount + + +def showRoleSummary(): + """Show a summary of roles grouped by type.""" + db = _getDbConnector() + + allRoles = db.getRecordset(Role, recordFilter=None) + + # Categorize roles + systemRoles = [] + templateRoles = [] + mandateRoles = [] + instanceRoles = [] + + for role in allRoles: + mandateId = role.get("mandateId") + featureInstanceId = role.get("featureInstanceId") + featureCode = role.get("featureCode") + isSystemRole = role.get("isSystemRole", False) + + if isSystemRole: + systemRoles.append(role) + elif mandateId is None and featureInstanceId is None and featureCode: + templateRoles.append(role) + elif mandateId is None and featureInstanceId is None and not featureCode: + # Global non-system role (shouldn't exist normally) + systemRoles.append(role) + elif mandateId and featureInstanceId is None: + mandateRoles.append(role) + elif featureInstanceId: + instanceRoles.append(role) + + print("\n" + "=" * 60) + print("ROLE SUMMARY") + print("=" * 60) + + print(f"\n1. SYSTEM ROLES ({len(systemRoles)}):") + for r in systemRoles: + print(f" - {r.get('roleLabel')} (isSystemRole={r.get('isSystemRole')})") + + print(f"\n2. TEMPLATE ROLES ({len(templateRoles)}) - (mandateId=NULL, featureInstanceId=NULL, featureCode!=NULL):") + templateByFeature = {} + for r in templateRoles: + fc = r.get("featureCode") + if fc not in templateByFeature: + templateByFeature[fc] = [] + templateByFeature[fc].append(r) + + for fc, roles in sorted(templateByFeature.items()): + print(f" [{fc}] ({len(roles)} roles):") + for r in roles: + print(f" - {r.get('roleLabel')}") + + print(f"\n3. MANDATE ROLES ({len(mandateRoles)}) - (mandateId!=NULL, featureInstanceId=NULL):") + for r in mandateRoles[:10]: # Show max 10 + print(f" - {r.get('roleLabel')} (mandate: {r.get('mandateId')[:8]}...)") + if len(mandateRoles) > 10: + print(f" ... and {len(mandateRoles) - 10} more") + + print(f"\n4. INSTANCE ROLES ({len(instanceRoles)}) - (featureInstanceId!=NULL):") + for r in instanceRoles[:10]: # Show max 10 + print(f" - {r.get('roleLabel')} (instance: {r.get('featureInstanceId')[:8]}...)") + if len(instanceRoles) > 10: + print(f" ... and {len(instanceRoles) - 10} more") + + print("\n" + "=" * 60) + print(f"TOTAL: {len(allRoles)} roles") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Cleanup duplicate roles in database") + parser.add_argument("--summary", action="store_true", help="Show role summary without deleting") + parser.add_argument("--cleanup", action="store_true", help="Delete duplicate roles") + + args = parser.parse_args() + + if args.summary: + showRoleSummary() + elif args.cleanup: + cleanupDuplicateRoles() + showRoleSummary() + else: + # Default: show summary only + showRoleSummary() + print("\nTo delete duplicates, run with --cleanup flag") diff --git a/tool_db_export_migration.py b/scripts/script_db_export_migration.py similarity index 97% rename from tool_db_export_migration.py rename to scripts/script_db_export_migration.py index aea697c1..dd3cc940 100644 --- a/tool_db_export_migration.py +++ b/scripts/script_db_export_migration.py @@ -16,7 +16,7 @@ Datenbanken: - poweron_trustee (Trustee Daten) Verwendung: - python tool_db_export_migration.py [--output ] [--pretty] + python script_db_export_migration.py [--output ] [--pretty] Optionen: --output, -o Pfad zur Ausgabedatei (Standard: migration_export_.json) @@ -40,23 +40,26 @@ from pathlib import Path # Find gateway directory (could be in local/pending/ or gateway/) scriptPath = Path(__file__).resolve() gatewayPath = scriptPath.parent +# If we're in scripts/, go up to gateway/ +if gatewayPath.name == "scripts": + gatewayPath = gatewayPath.parent # If we're in local/pending/, go up to find gateway/ -if gatewayPath.name == "pending": +elif gatewayPath.name == "pending": gatewayPath = gatewayPath.parent.parent / "gateway" elif gatewayPath.name == "local": gatewayPath = gatewayPath.parent / "gateway" # If gateway doesn't exist, try current directory if not gatewayPath.exists(): - gatewayPath = Path(__file__).parent.parent.parent / "gateway" + gatewayPath = scriptPath.parent.parent.parent / "gateway" if gatewayPath.exists(): sys.path.insert(0, str(gatewayPath)) # Change working directory to gateway so APP_CONFIG can find .env file os.chdir(str(gatewayPath)) else: # Fallback: assume we're already in gateway/ or add parent - sys.path.insert(0, str(Path(__file__).parent)) + sys.path.insert(0, str(scriptPath.parent)) # Try to change to gateway directory if it exists - potentialGateway = Path(__file__).parent + potentialGateway = scriptPath.parent if potentialGateway.exists() and (potentialGateway / "modules" / "shared" / "configuration.py").exists(): os.chdir(str(potentialGateway)) @@ -579,7 +582,7 @@ def exportDatabase( if logDir and os.path.isabs(logDir): outputDir = logDir else: - outputDir = os.path.join(os.path.dirname(__file__), "local", "logs") + outputDir = os.path.join(str(gatewayPath), "local", "logs") os.makedirs(outputDir, exist_ok=True) outputPath = os.path.join(outputDir, f"migration_export_{timestamp}.json") @@ -751,12 +754,12 @@ Datenbanken: poweron_trustee - Trustee Daten Beispiele: - python tool_db_export_migration.py - python tool_db_export_migration.py --pretty - python tool_db_export_migration.py -o backup.json --pretty - python tool_db_export_migration.py --db poweron_app,poweron_chat - python tool_db_export_migration.py --exclude Token,AuthEvent --include-meta - python tool_db_export_migration.py --summary + python script_db_export_migration.py + python script_db_export_migration.py --pretty + python script_db_export_migration.py -o backup.json --pretty + python script_db_export_migration.py --db poweron_app,poweron_chat + python script_db_export_migration.py --exclude Token,AuthEvent --include-meta + python script_db_export_migration.py --summary """ ) diff --git a/tool_security_encrypt_all_env_files.py b/scripts/script_security_encrypt_all_env_files.py similarity index 95% rename from tool_security_encrypt_all_env_files.py rename to scripts/script_security_encrypt_all_env_files.py index 420591dd..9981ad52 100644 --- a/tool_security_encrypt_all_env_files.py +++ b/scripts/script_security_encrypt_all_env_files.py @@ -10,16 +10,16 @@ keys for each environment. Usage: # Encrypt all secrets in all environment files - python tool_security_encrypt_all_env_files.py + python script_security_encrypt_all_env_files.py # Dry run - show what would be changed without making changes - python tool_security_encrypt_all_env_files.py --dry-run + python script_security_encrypt_all_env_files.py --dry-run # Skip backup creation - python tool_security_encrypt_all_env_files.py --no-backup + python script_security_encrypt_all_env_files.py --no-backup # Process only specific environment files - python tool_security_encrypt_all_env_files.py --files env_dev.env env_prod.env + python script_security_encrypt_all_env_files.py --files env_dev.env env_prod.env """ import sys @@ -29,14 +29,15 @@ from pathlib import Path from datetime import datetime from typing import List, Dict, Any -# Add the modules directory to the Python path -current_dir = Path(__file__).parent -modules_dir = current_dir / 'modules' -if modules_dir.exists(): - sys.path.insert(0, str(modules_dir)) +# Add the gateway directory to the Python path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +modulesDir = gatewayPath / "modules" +if modulesDir.exists(): + sys.path.insert(0, str(gatewayPath)) else: - print(f"Error: Modules directory not found: {modules_dir}") - print(f"Make sure you're running this script from the gateway directory") + print(f"Error: Modules directory not found: {modulesDir}") + print("Make sure you're running this script from the gateway directory") sys.exit(1) # Import encryption functions @@ -45,7 +46,7 @@ try: except ImportError as e: print(f"Error: Could not import encryption functions from shared.configuration: {e}") print(f"Make sure you're running this script from the gateway directory") - print(f"Modules directory: {modules_dir}") + print(f"Modules directory: {modulesDir}") sys.exit(1) def get_env_type_from_file(file_path: Path) -> str: diff --git a/tool_security_encrypt_config_value.py b/scripts/script_security_encrypt_config_value.py similarity index 96% rename from tool_security_encrypt_config_value.py rename to scripts/script_security_encrypt_config_value.py index e082e541..afaeb827 100644 --- a/tool_security_encrypt_config_value.py +++ b/scripts/script_security_encrypt_config_value.py @@ -10,18 +10,18 @@ It can also encrypt all *_SECRET keys in an environment file at once. Usage: # Encrypt a single value - python tool_encrypt_config_value.py --value "my_secret_value" --env dev - python tool_encrypt_config_value.py --file "path/to/file.json" --env prod + python script_security_encrypt_config_value.py --value "my_secret_value" --env dev + python script_security_encrypt_config_value.py --file "path/to/file.json" --env prod # Encrypt all secrets in a file - python tool_encrypt_config_value.py --encrypt-all env_dev.env --env dev - python tool_encrypt_config_value.py --encrypt-all env_prod.env --env prod --dry-run + python script_security_encrypt_config_value.py --encrypt-all env_dev.env --env dev + python script_security_encrypt_config_value.py --encrypt-all env_prod.env --env prod --dry-run # Decrypt a value (for testing) - python tool_encrypt_config_value.py --decrypt "DEV_ENC:encrypted_value" + python script_security_encrypt_config_value.py --decrypt "DEV_ENC:encrypted_value" # Verify master key is correct - python tool_encrypt_config_value.py --verify "PROD_ENC:Z0FBQUFBQm8xSU5p..." + python script_security_encrypt_config_value.py --verify "PROD_ENC:Z0FBQUFBQm8xSU5p..." """ import sys @@ -32,8 +32,11 @@ import shutil from pathlib import Path from datetime import datetime -# Add the modules directory to the Python path -sys.path.insert(0, str(Path(__file__).parent / 'modules')) +# Add the gateway directory to the Python path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +projectRoot = gatewayPath.parent +sys.path.insert(0, str(gatewayPath)) from modules.shared.configuration import encryptValue, decryptValue, _isEncryptedValue as isEncryptedValue @@ -412,7 +415,7 @@ def main(): key_source = f"file '{key_location}'" else: # Try default key file location - default_key_file = Path(__file__).parent.parent / 'local' / 'key.txt' + default_key_file = projectRoot / "local" / "key.txt" if default_key_file.exists(): print(f" [OK] Found master key in default file: {default_key_file}") key_source = f"file '{default_key_file}'" diff --git a/tool_security_generate_master_keys.py b/scripts/script_security_generate_master_keys.py similarity index 88% rename from tool_security_generate_master_keys.py rename to scripts/script_security_generate_master_keys.py index a5426e4f..6da55d26 100644 --- a/tool_security_generate_master_keys.py +++ b/scripts/script_security_generate_master_keys.py @@ -8,8 +8,8 @@ This tool generates cryptographically secure 256-bit master keys for all environ and updates the key.txt file with the new keys. Usage: - python generate_master_keys.py - python generate_master_keys.py --output "path/to/key.txt" + python script_security_generate_master_keys.py + python script_security_generate_master_keys.py --output "path/to/key.txt" """ import sys @@ -19,6 +19,10 @@ import base64 import argparse from pathlib import Path +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +projectRoot = gatewayPath.parent + def generate_master_key(): """Generate a secure 256-bit master key.""" # Generate 32 random bytes (256 bits) @@ -29,8 +33,8 @@ def generate_master_key(): def main(): parser = argparse.ArgumentParser(description='Generate secure master keys for all environments') parser.add_argument('--output', '-o', - default='../local/key.txt', - help='Output file path (default: ../local/key.txt)') + default=str(projectRoot / "local" / "key.txt"), + help='Output file path (default: poweron/local/key.txt)') parser.add_argument('--force', '-f', action='store_true', help='Overwrite existing key file without confirmation') diff --git a/tool_stats_durations_from_log.py b/scripts/script_stats_durations_from_log.py similarity index 100% rename from tool_stats_durations_from_log.py rename to scripts/script_stats_durations_from_log.py diff --git a/tool_stats_get_codelines.py b/scripts/script_stats_get_codelines.py similarity index 100% rename from tool_stats_get_codelines.py rename to scripts/script_stats_get_codelines.py diff --git a/tool_stats_showUnusedFunctions.py b/scripts/script_stats_showUnusedFunctions.py similarity index 96% rename from tool_stats_showUnusedFunctions.py rename to scripts/script_stats_showUnusedFunctions.py index 22dd5144..f7f6b2c3 100644 --- a/tool_stats_showUnusedFunctions.py +++ b/scripts/script_stats_showUnusedFunctions.py @@ -191,18 +191,19 @@ class FunctionUsageAnalyzer: def main(): """Main function to run the analysis.""" - # Get the directory where this script is located - script_dir = Path(__file__).parent - logger.info(f"Analyzing codebase in: {script_dir}") + # Get the gateway directory for a full codebase scan + scriptPath = Path(__file__).resolve() + gatewayPath = scriptPath.parent.parent + logger.info(f"Analyzing codebase in: {gatewayPath}") - analyzer = FunctionUsageAnalyzer(script_dir) + analyzer = FunctionUsageAnalyzer(gatewayPath) analyzer.analyze_codebase() report = analyzer.generate_report() print(report) # Save report to file - report_file = script_dir / "unused_functions_report.txt" + report_file = gatewayPath / "unused_functions_report.txt" with open(report_file, 'w', encoding='utf-8') as f: f.write(report)