From efe540b4f9c2cc2d617b6688095c02a628138d55 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 28 Mar 2026 16:59:01 +0100 Subject: [PATCH] fixing round 1 --- app.py | 3 + modules/datamodels/datamodelAi.py | 1 + .../datamodels/datamodelFeatureDataSource.py | 15 ++ modules/datamodels/datamodelKnowledge.py | 5 + modules/datamodels/datamodelUam.py | 6 + .../commcoach/routeFeatureCommcoach.py | 2 +- .../workspace/interfaceFeatureWorkspace.py | 2 +- .../workspace/routeFeatureWorkspace.py | 23 ++ modules/interfaces/interfaceBootstrap.py | 8 + modules/interfaces/interfaceDbApp.py | 248 +++++++++++------- modules/interfaces/interfaceDbManagement.py | 7 +- modules/interfaces/interfaceFeatures.py | 47 +++- modules/migration/migrateRootUsers.py | 129 ++++++++- modules/routes/routeAdminFeatures.py | 110 ++++++-- modules/routes/routeDataFiles.py | 49 ++++ modules/routes/routeDataSources.py | 97 +++++++ modules/routes/routeDataUsers.py | 9 +- modules/routes/routeSecurityLocal.py | 53 +++- modules/routes/routeStore.py | 60 ++--- modules/routes/routeSubscription.py | 58 +++- modules/routes/routeSystem.py | 4 +- .../services/serviceAi/mainServiceAi.py | 98 +++++-- .../serviceKnowledge/mainServiceKnowledge.py | 15 +- modules/shared/attributeUtils.py | 21 -- 24 files changed, 844 insertions(+), 226 deletions(-) create mode 100644 modules/routes/routeDataSources.py diff --git a/app.py b/app.py index 8268377a..63a18f94 100644 --- a/app.py +++ b/app.py @@ -545,6 +545,9 @@ app.include_router(userRouter) from modules.routes.routeDataFiles import router as fileRouter app.include_router(fileRouter) +from modules.routes.routeDataSources import router as dataSourceRouter +app.include_router(dataSourceRouter) + from modules.routes.routeDataPrompts import router as promptRouter app.include_router(promptRouter) diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py index 296500aa..c31d5696 100644 --- a/modules/datamodels/datamodelAi.py +++ b/modules/datamodels/datamodelAi.py @@ -168,6 +168,7 @@ class AiCallRequest(BaseModel): contentParts: Optional[List['ContentPart']] = None # Content parts for model-aware chunking messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="OpenAI-style messages for multi-turn agent conversations") tools: Optional[List[Dict[str, Any]]] = Field(default=None, description="Tool definitions for native function calling") + requireNeutralization: Optional[bool] = Field(default=None, description="Per-request neutralization override: True=force, False=skip, None=use config") class AiCallResponse(BaseModel): diff --git a/modules/datamodels/datamodelFeatureDataSource.py b/modules/datamodels/datamodelFeatureDataSource.py index 89b8b372..5aa834eb 100644 --- a/modules/datamodels/datamodelFeatureDataSource.py +++ b/modules/datamodels/datamodelFeatureDataSource.py @@ -25,6 +25,21 @@ class FeatureDataSource(BaseModel): userId: str = Field(default="", description="Owner user ID") workspaceInstanceId: str = Field(description="Workspace instance where this source is used") createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global", + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}}, + {"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}}, + {"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}}, + {"value": "global", "label": {"en": "Global", "de": "Global"}}, + ]} + ) + neutralize: bool = Field( + default=False, + description="Whether this data source should be neutralized before AI processing", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) registerModelLabels( diff --git a/modules/datamodels/datamodelKnowledge.py b/modules/datamodels/datamodelKnowledge.py index ac1c4ecc..e9dcc857 100644 --- a/modules/datamodels/datamodelKnowledge.py +++ b/modules/datamodels/datamodelKnowledge.py @@ -42,6 +42,10 @@ class FileContentIndex(BaseModel): default=None, description="Neutralization status: completed, failed, skipped, None = not required", ) + isNeutralized: bool = Field( + default=False, + description="True if content was neutralized before indexing", + ) registerModelLabels( @@ -64,6 +68,7 @@ registerModelLabels( "status": {"en": "Status", "fr": "Statut"}, "scope": {"en": "Scope", "de": "Sichtbarkeit"}, "neutralizationStatus": {"en": "Neutralization Status", "de": "Neutralisierungsstatus"}, + "isNeutralized": {"en": "Is Neutralized", "de": "Neutralisiert"}, }, ) diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index d56bd861..3e1250c7 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -103,6 +103,11 @@ class Mandate(BaseModel): {"value": "company", "label": {"en": "Company", "de": "Unternehmen"}}, ]} ) + deletedAt: Optional[float] = Field( + default=None, + description="Timestamp when the mandate was soft-deleted. After 30 days, hard-delete is triggered.", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) @field_validator('isSystem', mode='before') @classmethod @@ -135,6 +140,7 @@ registerModelLabels( "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, "isSystem": {"en": "System Mandate", "de": "System-Mandant", "fr": "Mandat système"}, "mandateType": {"en": "Mandate Type", "de": "Mandantentyp", "fr": "Type de mandat"}, + "deletedAt": {"en": "Deleted at", "de": "Gelöscht am", "fr": "Supprimé le"}, }, ) diff --git a/modules/features/commcoach/routeFeatureCommcoach.py b/modules/features/commcoach/routeFeatureCommcoach.py index 6d6eb44f..ccb4d342 100644 --- a/modules/features/commcoach/routeFeatureCommcoach.py +++ b/modules/features/commcoach/routeFeatureCommcoach.py @@ -2,7 +2,7 @@ # All rights reserved. """ CommCoach routes for the backend API. -Implements coaching context management, session streaming, tasks, dashboard, and voice endpoints. +Implements coaching context management, session streaming, tasks, and dashboard. """ import logging diff --git a/modules/features/workspace/interfaceFeatureWorkspace.py b/modules/features/workspace/interfaceFeatureWorkspace.py index 525ac62e..05bda01d 100644 --- a/modules/features/workspace/interfaceFeatureWorkspace.py +++ b/modules/features/workspace/interfaceFeatureWorkspace.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Interface for Workspace feature — manages VoiceSettings and WorkspaceUserSettings. +Interface for Workspace feature — manages WorkspaceUserSettings. Uses a dedicated poweron_workspace database. """ diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index 1828cba6..7698181a 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -893,6 +893,23 @@ async def listWorkspaceWorkflows( _validateInstanceAccess(instanceId, context) chatInterface = _getChatInterface(context, featureInstanceId=instanceId) workflows = chatInterface.getWorkflows() or [] + + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + _fiCache: Dict[str, Dict[str, str]] = {} + + def _resolveFeatureLabels(fiId: str) -> Dict[str, str]: + if fiId not in _fiCache: + fi = rootIf.getFeatureInstance(fiId) + if fi: + _fiCache[fiId] = { + "featureLabel": getattr(fi, "label", "") or getattr(fi, "featureCode", fiId), + "featureCode": getattr(fi, "featureCode", ""), + } + else: + _fiCache[fiId] = {"featureLabel": fiId[:8], "featureCode": ""} + return _fiCache[fiId] + items = [] for wf in workflows: if isinstance(wf, dict): @@ -904,9 +921,15 @@ async def listWorkspaceWorkflows( "status": getattr(wf, "status", ""), "startedAt": getattr(wf, "startedAt", None), "lastActivity": getattr(wf, "lastActivity", None), + "featureInstanceId": getattr(wf, "featureInstanceId", instanceId), } if not includeArchived and item.get("status") == "archived": continue + fiId = item.get("featureInstanceId") or instanceId + labels = _resolveFeatureLabels(fiId) + item.setdefault("featureLabel", labels["featureLabel"]) + item.setdefault("featureCode", labels["featureCode"]) + item.setdefault("featureInstanceId", fiId) items.append(item) return JSONResponse({"workflows": items}) diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index e2a0dfa4..0a3e24ad 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -133,6 +133,14 @@ def initBootstrap(db: DatabaseConnector) -> None: # Auto-provision Stripe Products/Prices for paid plans (idempotent) _bootstrapStripePrices() + # Purge soft-deleted mandates past 30-day retention + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rootIf.purgeExpiredMandates(retentionDays=30) + except Exception as e: + logger.warning(f"Mandate retention purge failed: {e}") + def initAutomationTemplates(dbApp: DatabaseConnector, adminUserId: Optional[str] = None) -> None: """ diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 6645e929..183bedb6 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -800,48 +800,6 @@ class AppObjects: logger.error(f"Error updating user: {str(e)}") raise ValueError(f"Failed to update user: {str(e)}") - def _assignUserToRootMandate(self, userId: str) -> None: - """ - Assign a new user to the root mandate with the mandate-instance 'user' role. - This ensures every user has a base membership in the system mandate. - - Uses the mandate-instance role (mandateId=rootMandateId), not the global template. - Feature instance access is NOT granted here - it is managed separately - via invitations or admin assignment. - - Args: - userId: User ID to assign - """ - try: - from modules.datamodels.datamodelRbac import Role - - rootMandateId = self._getRootMandateId() - if not rootMandateId: - logger.warning("No root mandate found, skipping root mandate assignment") - return - - # Check if user already has a mandate membership - existing = self.getUserMandate(userId, rootMandateId) - if existing: - logger.debug(f"User {userId} already assigned to root mandate") - return - - # Mandate-instance 'user' role (bound to this mandate, not a global template) - mandateUserRoles = self.db.getRecordset( - Role, - recordFilter={"roleLabel": "user", "mandateId": rootMandateId, "featureInstanceId": None} - ) - userRoleId = mandateUserRoles[0].get("id") if mandateUserRoles else None - - roleIds = [userRoleId] if userRoleId else [] - - self.createUserMandate(userId, rootMandateId, roleIds) - logger.info(f"Assigned user {userId} to root mandate with user role") - - except Exception as e: - # Log but don't fail user creation - logger.error(f"Error assigning user {userId} to root mandate: {e}") - def disableUser(self, userId: str) -> User: """Disables a user if current user has permission.""" return self.updateUser(userId, {"enabled": False}) @@ -1493,11 +1451,10 @@ class AppObjects: adminRoleId = r.get("id") break - userMandate = UserMandate(userId=userId, mandateId=mandateId, enabled=True) - createdUm = self.db.recordCreate(UserMandate, userMandate.model_dump()) - if adminRoleId and createdUm: - umRole = UserMandateRole(userMandateId=createdUm["id"], roleId=adminRoleId) - self.db.recordCreate(UserMandateRole, umRole.model_dump()) + if not adminRoleId: + raise ValueError(f"No admin role found for mandate {mandateId} — cannot assign user without role") + + self.createUserMandate(userId, mandateId, roleIds=[adminRoleId]) subscription = MandateSubscription( mandateId=mandateId, @@ -1533,14 +1490,16 @@ class AppObjects: instanceRoles = self.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) adminInstRoleId = None for ir in instanceRoles: - if "admin" in (ir.get("roleLabel") or "").lower(): + roleLabel = (ir.get("roleLabel") or "").lower() + if roleLabel.endswith("-admin"): adminInstRoleId = ir.get("id") break - fa = FeatureAccess(userId=userId, featureInstanceId=instanceId, enabled=True) - createdFa = self.db.recordCreate(FeatureAccess, fa.model_dump()) - if adminInstRoleId and createdFa: - far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminInstRoleId) - self.db.recordCreate(FeatureAccessRole, far.model_dump()) + if not adminInstRoleId: + raise ValueError( + f"No feature-specific admin role (e.g. {featureCode}-admin) for instance {instanceId}. " + f"Template roles not synced for feature '{featureCode}'." + ) + self.createFeatureAccess(userId, instanceId, roleIds=[adminInstRoleId]) except Exception as e: logger.error(f"Error auto-creating instance for '{featureName}': {e}") @@ -1669,15 +1628,72 @@ class AppObjects: raise PermissionError(f"No permission to delete mandate {mandateId}") if not force: - self.db.recordModify(Mandate, mandateId, {"enabled": False}) - logger.info(f"Soft-deleted mandate {mandateId}") + from modules.shared.timeUtils import getUtcTimestamp + self.db.recordModify(Mandate, mandateId, {"enabled": False, "deletedAt": getUtcTimestamp()}) + logger.info(f"Soft-deleted mandate {mandateId} (30-day retention)") return True # Hard delete with cascade from modules.datamodels.datamodelSubscription import MandateSubscription + from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog + from modules.datamodels.datamodelFiles import FileItem + from modules.datamodels.datamodelDataSource import DataSource + from modules.datamodels.datamodelKnowledge import FileContentIndex + from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutralizerAttributes + + instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) + + # 0. Delete instance-scoped data for each FeatureInstance + for inst in instances: + instId = inst.get("id") + if not instId: + continue + + # 0a. FileContentIndex (knowledge/RAG) + fciRecords = self.db.getRecordset(FileContentIndex, recordFilter={"featureInstanceId": instId}) + for rec in fciRecords: + self.db.recordDelete(FileContentIndex, rec.get("id")) + if fciRecords: + logger.info(f"Cascade: deleted {len(fciRecords)} FileContentIndex records for instance {instId}") + + # 0b. DataNeutralizerAttributes + dnaRecords = self.db.getRecordset(DataNeutralizerAttributes, recordFilter={"featureInstanceId": instId}) + for rec in dnaRecords: + self.db.recordDelete(DataNeutralizerAttributes, rec.get("id")) + if dnaRecords: + logger.info(f"Cascade: deleted {len(dnaRecords)} DataNeutralizerAttributes for instance {instId}") + + # 0c. DataSource + dsRecords = self.db.getRecordset(DataSource, recordFilter={"featureInstanceId": instId}) + for rec in dsRecords: + self.db.recordDelete(DataSource, rec.get("id")) + if dsRecords: + logger.info(f"Cascade: deleted {len(dsRecords)} DataSource records for instance {instId}") + + # 0d. FileItem + fileRecords = self.db.getRecordset(FileItem, recordFilter={"featureInstanceId": instId}) + for rec in fileRecords: + self.db.recordDelete(FileItem, rec.get("id")) + if fileRecords: + logger.info(f"Cascade: deleted {len(fileRecords)} FileItem records for instance {instId}") + + # 0e. ChatWorkflow + ChatMessage + ChatLog + workflows = self.db.getRecordset(ChatWorkflow, recordFilter={"featureInstanceId": instId}) + for wf in workflows: + wfId = wf.get("id") + if not wfId: + continue + msgs = self.db.getRecordset(ChatMessage, recordFilter={"workflowId": wfId}) + for msg in msgs: + self.db.recordDelete(ChatMessage, msg.get("id")) + logs = self.db.getRecordset(ChatLog, recordFilter={"workflowId": wfId}) + for log in logs: + self.db.recordDelete(ChatLog, log.get("id")) + self.db.recordDelete(ChatWorkflow, wfId) + if workflows: + logger.info(f"Cascade: deleted {len(workflows)} ChatWorkflows (with messages/logs) for instance {instId}") # 1. Delete FeatureAccess + FeatureAccessRole for all instances in this mandate - instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) for inst in instances: instId = inst.get("id") accesses = self.db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instId}) @@ -1692,10 +1708,20 @@ class AppObjects: self.db.recordDelete(UserMandate, um.get("id")) logger.info(f"Cascade: deleted {len(memberships)} UserMandates for mandate {mandateId}") - # 3. Delete MandateSubscriptions + # 3. Cancel Stripe subscriptions + delete MandateSubscription records subs = self.db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId}) for sub in subs: - self.db.recordDelete(MandateSubscription, sub.get("id")) + subId = sub.get("id") + stripeSubId = sub.get("stripeSubscriptionId") + if stripeSubId: + try: + from modules.shared.stripeClient import getStripeClient + stripe = getStripeClient() + stripe.Subscription.cancel(stripeSubId) + logger.info(f"Cancelled Stripe subscription {stripeSubId} for mandate {mandateId}") + except Exception as e: + logger.warning(f"Failed to cancel Stripe sub {stripeSubId}: {e}") + self.db.recordDelete(MandateSubscription, subId) logger.info(f"Cascade: deleted {len(subs)} subscriptions for mandate {mandateId}") # 4. Delete mandate-level Roles @@ -1717,6 +1743,35 @@ class AppObjects: logger.error(f"Error deleting mandate: {str(e)}") raise ValueError(f"Failed to delete mandate: {str(e)}") + def restoreMandate(self, mandateId: str) -> bool: + """Restore a soft-deleted mandate (undo soft-delete within the 30-day retention window).""" + mandate = self.getMandate(mandateId) + if not mandate: + return False + self.db.recordModify(Mandate, mandateId, {"enabled": True, "deletedAt": None}) + logger.info(f"Restored soft-deleted mandate {mandateId}") + return True + + def purgeExpiredMandates(self, retentionDays: int = 30) -> int: + """Hard-delete all mandates whose soft-delete timestamp exceeds the retention period.""" + import time + cutoff = time.time() - (retentionDays * 86400) + allMandates = self.db.getRecordset(Mandate) + purged = 0 + for m in allMandates: + deletedAt = m.get("deletedAt") if isinstance(m, dict) else getattr(m, "deletedAt", None) + enabled = m.get("enabled") if isinstance(m, dict) else getattr(m, "enabled", True) + mandateId = m.get("id") if isinstance(m, dict) else getattr(m, "id", None) + if deletedAt and not enabled and deletedAt < cutoff and mandateId: + try: + self.deleteMandate(mandateId, force=True) + purged += 1 + except Exception as e: + logger.error(f"Failed to purge expired mandate {mandateId}: {e}") + if purged: + logger.info(f"Purged {purged} expired mandate(s) beyond {retentionDays}-day retention") + return purged + # ============================================ # User-Mandate Membership Methods (Multi-Tenant) # ============================================ @@ -1774,45 +1829,44 @@ class AppObjects: Create a UserMandate record (add user to mandate). Also creates a billing account for the user if billing is configured for PREPAY_USER. + INVARIANT: A UserMandate MUST have at least one UserMandateRole. + Args: userId: User ID mandateId: Mandate ID - roleIds: Optional list of role IDs to assign + roleIds: List of role IDs to assign (at least one required) Returns: Created UserMandate object """ + if not roleIds: + raise ValueError(f"Cannot create UserMandate without roles for user {userId} in mandate {mandateId}") + try: - # Check if already exists existing = self.getUserMandate(userId, mandateId) if existing: raise ValueError(f"User {userId} is already member of mandate {mandateId}") - # Subscription capacity check (before insert) self._checkSubscriptionCapacity(mandateId, "users", delta=1) - # Create UserMandate userMandate = UserMandate( userId=userId, mandateId=mandateId, enabled=True ) createdRecord = self.db.recordCreate(UserMandate, userMandate.model_dump()) + if not createdRecord: + raise ValueError("Database failed to create UserMandate record") - # Assign roles via junction table - if roleIds and createdRecord: - userMandateId = createdRecord.get("id") - for roleId in roleIds: - userMandateRole = UserMandateRole( - userMandateId=userMandateId, - roleId=roleId - ) - self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) + userMandateId = createdRecord.get("id") + for roleId in roleIds: + userMandateRole = UserMandateRole( + userMandateId=userMandateId, + roleId=roleId + ) + self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) - # Create billing account for user if billing is configured self._ensureUserBillingAccount(userId, mandateId) - - # Sync Stripe quantity after successful insert self._syncSubscriptionQuantity(mandateId) cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} @@ -2198,40 +2252,42 @@ class AppObjects: Create a FeatureAccess record (grant user access to feature instance). Also auto-assigns the user to the mandate with the 'user' role if not already a member. + INVARIANT: A FeatureAccess MUST have at least one FeatureAccessRole. + Args: userId: User ID featureInstanceId: FeatureInstance ID - roleIds: Optional list of role IDs to assign + roleIds: List of role IDs to assign (at least one required) Returns: Created FeatureAccess object """ + if not roleIds: + raise ValueError(f"Cannot create FeatureAccess without roles for user {userId} on instance {featureInstanceId}") + try: - # Check if already exists existing = self.getFeatureAccess(userId, featureInstanceId) if existing: raise ValueError(f"User {userId} already has access to feature instance {featureInstanceId}") - # Auto-assign user to mandate with 'user' role if not already a member self._ensureUserMandateMembership(userId, featureInstanceId) - # Create FeatureAccess featureAccess = FeatureAccess( userId=userId, featureInstanceId=featureInstanceId, enabled=True ) createdRecord = self.db.recordCreate(FeatureAccess, featureAccess.model_dump()) + if not createdRecord: + raise ValueError("Database failed to create FeatureAccess record") - # Assign roles via junction table - if roleIds and createdRecord: - featureAccessId = createdRecord.get("id") - for roleId in roleIds: - featureAccessRole = FeatureAccessRole( - featureAccessId=featureAccessId, - roleId=roleId - ) - self.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + featureAccessId = createdRecord.get("id") + for roleId in roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + self.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} return FeatureAccess(**cleanedRecord) @@ -2242,7 +2298,7 @@ class AppObjects: def _ensureUserMandateMembership(self, userId: str, featureInstanceId: str) -> None: """ Ensure user is a member of the mandate that owns the feature instance. - If not already a member, adds them with the 'user' role (no access rights, membership only). + If not already a member, adds them with the 'user' role. """ try: from modules.interfaces.interfaceFeatures import getFeatureInterface @@ -2255,28 +2311,30 @@ class AppObjects: mandateId = str(instance.mandateId) - # Check if user already has mandate membership existing = self.getUserMandate(userId, mandateId) if existing: logger.debug(f"User {userId} already member of mandate {mandateId}") return - # Find the mandate-level 'user' role (membership marker, no access rights) userRoles = self.db.getRecordset( Role, recordFilter={"roleLabel": "user", "mandateId": mandateId, "featureInstanceId": None} ) userRoleId = userRoles[0].get("id") if userRoles else None - roleIds = [userRoleId] if userRoleId else [] + if not userRoleId: + raise ValueError(f"No 'user' role found for mandate {mandateId} — cannot assign user without role") - self.createUserMandate(userId, mandateId, roleIds) + self.createUserMandate(userId, mandateId, roleIds=[userRoleId]) logger.info(f"Auto-assigned user {userId} to mandate {mandateId} with 'user' role (via feature instance {featureInstanceId})") - except ValueError: - # createUserMandate raises ValueError if already exists - safe to ignore - pass + except ValueError as ve: + if "already member" in str(ve): + pass + else: + raise except Exception as e: logger.error(f"Error auto-assigning user {userId} to mandate: {e}") + raise def getRoleIdsForFeatureAccess(self, featureAccessId: str) -> List[str]: """ diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 64883b95..58fd6926 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -940,7 +940,12 @@ class ComponentObjects: fileName = file.get("fileName") if not fileName or fileName == "None": continue - + + if file.get("scope") is None: + file["scope"] = "personal" + if file.get("neutralize") is None: + file["neutralize"] = False + fileItem = FileItem(**file) fileItems.append(fileItem) except Exception as e: diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py index b261e76e..56311f01 100644 --- a/modules/interfaces/interfaceFeatures.py +++ b/modules/interfaces/interfaceFeatures.py @@ -208,7 +208,11 @@ class FeatureInterface: def _copyTemplateRoles(self, featureCode: str, mandateId: str, instanceId: str) -> int: """ - Copy global template roles for a feature to a new instance. + Copy feature-specific template roles to a new instance. + + INVARIANT: Feature instances MUST receive feature-specific roles + (e.g. workspace-admin, workspace-user). NEVER generic mandate roles. + Feature templates have featureCode set and isSystemRole=False. Args: featureCode: Feature code @@ -217,19 +221,30 @@ class FeatureInterface: Returns: Number of roles copied + + Raises: + ValueError: If no feature-specific template roles exist """ try: - # Find global template roles for this feature (mandateId=None) - globalRoles = self.db.getRecordset( + allTemplates = self.db.getRecordset( Role, - recordFilter={"featureCode": featureCode, "mandateId": None} + recordFilter={"featureCode": featureCode} ) - if not globalRoles: - logger.debug(f"No template roles found for feature {featureCode}") - return 0 + featureTemplates = [ + r for r in allTemplates + if r.get("mandateId") is None and r.get("featureInstanceId") is None + ] - templateRoleIds = [r.get("id") for r in globalRoles] + if not featureTemplates: + raise ValueError( + f"No feature-specific template roles found for '{featureCode}'. " + f"Each feature module must define TEMPLATE_ROLES and sync them to DB on startup." + ) + + logger.info(f"Found {len(featureTemplates)} feature-specific template roles for '{featureCode}'") + + templateRoleIds = [r.get("id") for r in featureTemplates] # BULK: Load all template AccessRules in one query allTemplateRules = [] @@ -246,7 +261,7 @@ class FeatureInterface: # Copy roles and their AccessRules copiedCount = 0 - for templateRole in globalRoles: + for templateRole in featureTemplates: newRoleId = str(uuid.uuid4()) # Create new role for this instance @@ -282,9 +297,11 @@ class FeatureInterface: logger.info(f"Copied {copiedCount} template roles for instance {instanceId}") return copiedCount + except ValueError: + raise except Exception as e: logger.error(f"Error copying template roles: {e}") - return 0 + raise ValueError(f"Failed to copy template roles for '{featureCode}': {e}") def syncRolesFromTemplate(self, featureInstanceId: str, addOnly: bool = True) -> Dict[str, int]: """ @@ -309,11 +326,15 @@ class FeatureInterface: featureCode = instance.featureCode mandateId = instance.mandateId - # Get current template roles - templateRoles = self.db.getRecordset( + # Get feature-specific template roles (mandateId=None, featureInstanceId=None) + allForFeature = self.db.getRecordset( Role, - recordFilter={"featureCode": featureCode, "mandateId": None} + recordFilter={"featureCode": featureCode} ) + templateRoles = [ + r for r in allForFeature + if r.get("mandateId") is None and r.get("featureInstanceId") is None + ] templateLabels = {r.get("roleLabel") for r in templateRoles} # Get current instance roles diff --git a/modules/migration/migrateRootUsers.py b/modules/migration/migrateRootUsers.py index f1a55d9e..a048e614 100644 --- a/modules/migration/migrateRootUsers.py +++ b/modules/migration/migrateRootUsers.py @@ -7,12 +7,20 @@ Called once from bootstrap, sets a DB flag to prevent re-execution. """ import logging -from typing import Optional +from typing import Optional, List, Dict, Any logger = logging.getLogger(__name__) _MIGRATION_FLAG_KEY = "migration_root_users_completed" +_DATA_TABLES = [ + "ChatWorkflow", + "FileItem", + "DataSource", + "DataNeutralizerAttributes", + "FileContentIndex", +] + def _isMigrationCompleted(db) -> bool: """Check if migration has already been executed.""" @@ -37,6 +45,95 @@ def _setMigrationCompleted(db) -> None: logger.error(f"Failed to set migration flag: {e}") +def _findOrCreateTargetInstance(db, featureInterface, featureCode: str, targetMandateId: str, rootInstance: dict) -> dict: + """Find existing or create new FeatureInstance in target mandate. Idempotent.""" + from modules.datamodels.datamodelFeatures import FeatureInstance + + existing = db.getRecordset(FeatureInstance, recordFilter={ + "featureCode": featureCode, + "mandateId": targetMandateId, + }) + if existing: + logger.debug(f"Target instance already exists for {featureCode} in mandate {targetMandateId}") + return existing[0] + + label = rootInstance.get("label") or featureCode + instance = featureInterface.createFeatureInstance( + featureCode=featureCode, + mandateId=targetMandateId, + label=label, + enabled=True, + copyTemplateRoles=True, + ) + if isinstance(instance, dict): + return instance + return instance.model_dump() if hasattr(instance, "model_dump") else {"id": instance.id} + + +def _migrateDataRecords(db, oldInstanceId: str, newInstanceId: str, userId: str) -> int: + """Bulk-update featureInstanceId on all data tables for records owned by userId.""" + totalMigrated = 0 + db._ensure_connection() + for tableName in _DATA_TABLES: + try: + with db.connection.cursor() as cursor: + cursor.execute( + f'UPDATE "{tableName}" ' + f'SET "featureInstanceId" = %s ' + f'WHERE "featureInstanceId" = %s AND "_createdBy" = %s', + (newInstanceId, oldInstanceId, userId), + ) + count = cursor.rowcount + db.connection.commit() + if count > 0: + logger.info(f" Migrated {count} rows in {tableName}: {oldInstanceId} -> {newInstanceId}") + totalMigrated += count + except Exception as e: + try: + db.connection.rollback() + except Exception: + pass + logger.debug(f" Table {tableName} skipped (may not exist or no matching column): {e}") + return totalMigrated + + +def _grantFeatureAccess(db, userId: str, featureInstanceId: str) -> dict: + """Create FeatureAccess + admin role on a feature instance. Idempotent.""" + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + from modules.datamodels.datamodelRbac import Role + + existing = db.getRecordset(FeatureAccess, recordFilter={ + "userId": userId, + "featureInstanceId": featureInstanceId, + }) + if existing: + logger.debug(f"FeatureAccess already exists for user {userId} on instance {featureInstanceId}") + return existing[0] + + fa = FeatureAccess(userId=userId, featureInstanceId=featureInstanceId, enabled=True) + createdFa = db.recordCreate(FeatureAccess, fa.model_dump()) + if not createdFa: + logger.warning(f"Failed to create FeatureAccess for user {userId} on instance {featureInstanceId}") + return {} + + instanceRoles = db.getRecordset(Role, recordFilter={"featureInstanceId": featureInstanceId}) + adminRoleId = None + for r in instanceRoles: + roleLabel = (r.get("roleLabel") or "").lower() + if roleLabel.endswith("-admin"): + adminRoleId = r.get("id") + break + if not adminRoleId: + raise ValueError( + f"No feature-specific admin role for instance {featureInstanceId}. " + f"Cannot create FeatureAccess without role — even in migration context." + ) + far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminRoleId) + db.recordCreate(FeatureAccessRole, far.model_dump()) + + return createdFa + + def migrateRootUsers(db, dryRun: bool = False) -> dict: """ Migrate all end-user feature data from Root mandate to personal mandates. @@ -68,12 +165,15 @@ def migrateRootUsers(db, dryRun: bool = False) -> dict: ) from modules.datamodels.datamodelFeatures import FeatureInstance from modules.interfaces.interfaceDbApp import getRootInterface + from modules.interfaces.interfaceFeatures import getFeatureInterface rootInterface = getRootInterface() + featureInterface = getFeatureInterface(db) stats = { "usersProcessed": 0, "mandatesCreated": 0, "instancesMigrated": 0, + "dataRowsMigrated": 0, "rootInstancesDeleted": 0, "rootMembershipsRemoved": 0, "dryRun": dryRun, @@ -167,12 +267,29 @@ def migrateRootUsers(db, dryRun: bool = False) -> dict: logger.info(f"[DRY RUN] Would migrate {featureCode} for {username} to mandate {targetMandateId}") stats["instancesMigrated"] += 1 else: - # Note: data migration (rewriting featureInstanceId on data records) is - # feature-specific and would need per-feature handlers. For now, we create - # the new instance and transfer the access. Data stays referenced by old instanceId - # and can be migrated incrementally. - logger.info(f"Migrated access for {username} on {featureCode} (data migration deferred)") + targetInstance = _findOrCreateTargetInstance( + db, featureInterface, featureCode, targetMandateId, instRecords[0], + ) + newInstanceId = targetInstance.get("id") + if not newInstanceId: + logger.error(f"Failed to obtain target instance for {featureCode} in mandate {targetMandateId}") + continue + + migratedCount = _migrateDataRecords(db, oldInstanceId, newInstanceId, userId) + + _grantFeatureAccess(db, userId, newInstanceId) + + try: + db.recordDelete(FeatureAccess, oldAccessId) + except Exception as delErr: + logger.warning(f"Could not remove old FeatureAccess {oldAccessId}: {delErr}") + + logger.info( + f"Migrated {featureCode} for {username}: " + f"instance {oldInstanceId} -> {newInstanceId}, {migratedCount} data rows moved" + ) stats["instancesMigrated"] += 1 + stats["dataRowsMigrated"] += migratedCount stats["usersProcessed"] += 1 diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index c95c0b1b..12206b06 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -1172,31 +1172,29 @@ def add_user_to_feature_instance( detail=f"User '{data.userId}' not found" ) - # Check if user already has access - from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole - existingAccess = rootInterface.getFeatureAccess(data.userId, instanceId) - if existingAccess: + if not data.roleIds: raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="User already has access to this feature instance" + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one role is required to grant feature access" ) - - # Create FeatureAccess record - featureAccess = FeatureAccess( + + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) + validRoleIds = {r.get("id") for r in instanceRoles} + invalidRoles = [rid for rid in data.roleIds if rid not in validRoleIds] + if invalidRoles: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role IDs {invalidRoles} do not belong to feature instance {instanceId}. " + f"Only instance-scoped roles are allowed, never mandate roles." + ) + + featureAccess = rootInterface.createFeatureAccess( userId=data.userId, featureInstanceId=instanceId, - enabled=True + roleIds=data.roleIds ) - createdAccess = rootInterface.db.recordCreate(FeatureAccess, featureAccess.model_dump()) - featureAccessId = createdAccess.get("id") - - # Create FeatureAccessRole records for each role - for roleId in data.roleIds: - featureAccessRole = FeatureAccessRole( - featureAccessId=featureAccessId, - roleId=roleId - ) - rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + featureAccessId = str(featureAccess.id) logger.info( f"User {context.user.id} added user {data.userId} to feature instance {instanceId} " @@ -1379,10 +1377,19 @@ def update_feature_instance_user_roles( if data.enabled is not None: rootInterface.db.recordModify(FeatureAccess, featureAccessId, {"enabled": data.enabled}) - # Delete existing FeatureAccessRole records via interface method + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) + validRoleIds = {r.get("id") for r in instanceRoles} + invalidRoles = [rid for rid in data.roleIds if rid not in validRoleIds] + if invalidRoles: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role IDs {invalidRoles} do not belong to feature instance {instanceId}. " + f"Only instance-scoped roles are allowed, never mandate roles." + ) + rootInterface.deleteFeatureAccessRoles(featureAccessId) - # Create new FeatureAccessRole records for roleId in data.roleIds: featureAccessRole = FeatureAccessRole( featureAccessId=featureAccessId, @@ -1523,6 +1530,65 @@ def get_feature( ) +# ============================================================================= +# Instance Rename (for instance admins, used by navigation tree) +# ============================================================================= + +class FeatureInstanceRenameRequest(BaseModel): + """Request model for renaming a feature instance""" + label: str = Field(..., min_length=1, max_length=200, description="New label for the instance") + + +@router.patch("/instances/{instanceId}/rename", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +def _renameFeatureInstance( + request: Request, + instanceId: str, + data: FeatureInstanceRenameRequest, + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """ + Rename a feature instance. Requires instance admin role. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Feature instance not found") + + userId = str(context.user.id) + isInstanceAdmin = False + if context.hasSysAdminRole: + isInstanceAdmin = True + else: + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + fa = rootInterface.getFeatureAccess(userId, instanceId) + if fa: + faRoleIds = rootInterface.getRoleIdsForFeatureAccess(str(fa.id)) + for rid in faRoleIds: + role = rootInterface.getRole(rid) + if role and (role.roleLabel or "").lower().endswith("-admin"): + isInstanceAdmin = True + break + + if not isInstanceAdmin: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Instance admin role required to rename") + + updated = featureInterface.updateFeatureInstance(instanceId, {"label": data.label.strip()}) + if not updated: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update instance") + + return {"id": instanceId, "label": updated.label} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error renaming feature instance {instanceId}: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index f98b2306..5f71bb47 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -8,6 +8,7 @@ import json # Import auth module from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext +from modules.auth.authentication import _hasSysAdminRole # Import interfaces import modules.interfaces.interfaceDbManagement as interfaceDbManagement @@ -699,6 +700,20 @@ def updateFileScope( except Exception as e: logger.warning(f"Failed to update FileContentIndex scope for file {fileId}: {e}") + # Trigger re-indexing so RAG embeddings metadata reflects the new scope + try: + fileMeta = managementInterface.getFile(fileId) + if fileMeta: + import asyncio + asyncio.ensure_future(_autoIndexFile( + fileId=fileId, + fileName=fileMeta.fileName if hasattr(fileMeta, "fileName") else fileMeta.get("fileName", ""), + mimeType=fileMeta.mimeType if hasattr(fileMeta, "mimeType") else fileMeta.get("mimeType", ""), + user=context.user, + )) + except Exception as e: + logger.warning(f"Failed to trigger re-index after scope change for file {fileId}: {e}") + return {"fileId": fileId, "scope": scope, "updated": True} except HTTPException: raise @@ -725,6 +740,34 @@ def updateFileNeutralize( managementInterface.updateFile(fileId, {"neutralize": neutralize}) + # Update FileContentIndex neutralization metadata + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface + from modules.datamodels.datamodelKnowledge import FileContentIndex + knowledgeDb = getKnowledgeInterface() + neutralizationStatus = "neutralized" if neutralize else "original" + indices = knowledgeDb.db.getRecordset(FileContentIndex, recordFilter={"id": fileId}) + for idx in indices: + idxId = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if idxId: + knowledgeDb.db.recordModify(FileContentIndex, idxId, {"neutralizationStatus": neutralizationStatus}) + except Exception as e: + logger.warning(f"Failed to update FileContentIndex neutralize for file {fileId}: {e}") + + # Trigger re-indexing so content is re-processed with/without neutralization + try: + fileMeta = managementInterface.getFile(fileId) + if fileMeta: + import asyncio + asyncio.ensure_future(_autoIndexFile( + fileId=fileId, + fileName=fileMeta.fileName if hasattr(fileMeta, "fileName") else fileMeta.get("fileName", ""), + mimeType=fileMeta.mimeType if hasattr(fileMeta, "mimeType") else fileMeta.get("mimeType", ""), + user=context.user, + )) + except Exception as e: + logger.warning(f"Failed to trigger re-index after neutralize change for file {fileId}: {e}") + return {"fileId": fileId, "neutralize": neutralize, "updated": True} except Exception as e: logger.error(f"Error updating file neutralize flag: {e}") @@ -799,6 +842,12 @@ def update_file( detail=f"File with ID {fileId} not found" ) + if file_info.get("scope") == "global" and not _hasSysAdminRole(str(currentUser.id)): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only sysadmins can set global scope", + ) + # Check if user has access to the file using RBAC if not managementInterface.checkRbacPermission(FileItem, "update", fileId): raise HTTPException( diff --git a/modules/routes/routeDataSources.py b/modules/routes/routeDataSources.py new file mode 100644 index 00000000..e210d094 --- /dev/null +++ b/modules/routes/routeDataSources.py @@ -0,0 +1,97 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""PATCH endpoints for DataSource and FeatureDataSource scope/neutralize tagging.""" + +import logging +from typing import Any, Dict + +from fastapi import APIRouter, HTTPException, Depends, Path, Request, Body +from modules.auth import limiter, getRequestContext, RequestContext +from modules.auth.authentication import _hasSysAdminRole +from modules.datamodels.datamodelDataSource import DataSource +from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/datasources", + tags=["Data Sources"], + responses={ + 404: {"description": "Not found"}, + 400: {"description": "Bad request"}, + 401: {"description": "Unauthorized"}, + 403: {"description": "Forbidden"}, + 500: {"description": "Internal server error"}, + }, +) + +_VALID_SCOPES = {"personal", "featureInstance", "mandate", "global"} + + +def _findSourceRecord(db, sourceId: str): + """Look up a source by ID, checking DataSource first, then FeatureDataSource.""" + rec = db.getRecord(DataSource, sourceId) + if rec: + return rec, DataSource + rec = db.getRecord(FeatureDataSource, sourceId) + if rec: + return rec, FeatureDataSource + return None, None + + +@router.patch("/{sourceId}/scope") +@limiter.limit("30/minute") +def _updateDataSourceScope( + request: Request, + sourceId: str = Path(..., description="ID of the DataSource or FeatureDataSource"), + scope: str = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Update the scope of a DataSource or FeatureDataSource. Global scope requires sysAdmin.""" + if scope not in _VALID_SCOPES: + raise HTTPException(status_code=400, detail=f"Invalid scope: {scope}. Must be one of {_VALID_SCOPES}") + + if scope == "global" and not _hasSysAdminRole(context.user): + raise HTTPException(status_code=403, detail="Only sysadmins can set global scope") + + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rec, model = _findSourceRecord(rootIf.db, sourceId) + if not rec: + raise HTTPException(status_code=404, detail=f"DataSource {sourceId} not found") + + rootIf.db.recordModify(model, sourceId, {"scope": scope}) + logger.info("Updated scope=%s for %s %s", scope, model.__name__, sourceId) + return {"sourceId": sourceId, "scope": scope, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error("Error updating datasource scope: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch("/{sourceId}/neutralize") +@limiter.limit("30/minute") +def _updateDataSourceNeutralize( + request: Request, + sourceId: str = Path(..., description="ID of the DataSource or FeatureDataSource"), + neutralize: bool = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Toggle the neutralization flag on a DataSource or FeatureDataSource.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rec, model = _findSourceRecord(rootIf.db, sourceId) + if not rec: + raise HTTPException(status_code=404, detail=f"DataSource {sourceId} not found") + + rootIf.db.recordModify(model, sourceId, {"neutralize": neutralize}) + logger.info("Updated neutralize=%s for %s %s", neutralize, model.__name__, sourceId) + return {"sourceId": sourceId, "neutralize": neutralize, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error("Error updating datasource neutralize: %s", e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 7e903466..a1da658b 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -639,14 +639,17 @@ def create_user( # MULTI-TENANT: Add user to current mandate via UserMandate with default "user" role if context.mandateId: - # Get "user" role ID userRole = appInterface.getRoleByLabel("user") - roleIds = [str(userRole.id)] if userRole else [] + if not userRole: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No 'user' role found in system — cannot assign user to mandate" + ) appInterface.createUserMandate( userId=str(newUser.id), mandateId=str(context.mandateId), - roleIds=roleIds + roleIds=[str(userRole.id)] ) logger.info(f"Created UserMandate for user {newUser.id} in mandate {context.mandateId}") diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 8b1d9e8e..f066fda2 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -4,7 +4,7 @@ Routes for local security and authentication. """ -from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body, Query +from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body, Query, Path from fastapi.security import OAuth2PasswordRequestForm import logging from typing import Dict, Any @@ -14,7 +14,7 @@ import uuid from jose import jwt # Import auth modules -from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM +from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM, getRequestContext, RequestContext from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate, MandateType @@ -730,6 +730,13 @@ def onboarding_provision( planKey=planKey, ) + try: + activatedCount = appInterface._activatePendingSubscriptions(str(currentUser.id)) + if activatedCount > 0: + logger.info(f"Activated {activatedCount} pending subscription(s) for user {currentUser.username} during onboarding") + except Exception as subErr: + logger.error(f"Error activating subscriptions during onboarding: {subErr}") + logger.info(f"Onboarding provision for {currentUser.username}: {result}") return { "message": "Mandate provisioned successfully", @@ -922,3 +929,45 @@ async def testVoice( ).decode() return {"success": True, "audio": audioB64, "format": "mp3", "text": text} return {"success": False, "error": "TTS returned no audio"} + + +# ============================================================ +# Neutralization Mappings (user-level, view/delete) +# ============================================================ + +@router.get("/neutralization-mappings") +@limiter.limit("60/minute") +def _getNeutralizationMappings( + request: Request, + context: RequestContext = Depends(getRequestContext), +): + """List the current user's neutralization placeholder mappings.""" + userId = str(context.user.id) + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutralizerAttributes + rootIf = getRootInterface() + records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"userId": userId}) + return {"mappings": records} + + +@router.delete("/neutralization-mappings/{mappingId}") +@limiter.limit("30/minute") +def _deleteNeutralizationMapping( + request: Request, + mappingId: str = Path(..., description="ID of the mapping to delete"), + context: RequestContext = Depends(getRequestContext), +): + """Delete a specific neutralization mapping owned by the current user.""" + userId = str(context.user.id) + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutralizerAttributes + rootIf = getRootInterface() + records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"id": mappingId}) + if not records: + raise HTTPException(status_code=404, detail="Mapping not found") + rec = records[0] + recUserId = rec.get("userId") if isinstance(rec, dict) else getattr(rec, "userId", None) + if recUserId != userId: + raise HTTPException(status_code=403, detail="Not your mapping") + rootIf.db.recordDelete(DataNeutralizerAttributes, mappingId) + return {"deleted": True, "id": mappingId} diff --git a/modules/routes/routeStore.py b/modules/routes/routeStore.py index 99c582c6..cbd4ef6e 100644 --- a/modules/routes/routeStore.py +++ b/modules/routes/routeStore.py @@ -36,7 +36,7 @@ router = APIRouter( class StoreActivateRequest(BaseModel): """Request model for activating a store feature.""" featureCode: str = Field(..., description="Feature code to activate") - mandateId: Optional[str] = Field(None, description="Target mandate ID (explicit). If None and user has no admin mandate, auto-creates personal mandate.") + mandateId: str = Field(..., description="Target mandate ID — always explicit, never optional") class StoreDeactivateRequest(BaseModel): @@ -134,12 +134,27 @@ def listUserMandates( request: Request, context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: - """List mandates where the user can activate features (admin mandates).""" + """ + List mandates where the user can activate features (admin mandates). + If user has 0 admin mandates, auto-provisions a personal mandate so the + Store always has a clear mandate context. + """ try: rootInterface = getRootInterface() db = rootInterface.db userId = str(context.user.id) adminMandateIds = _getUserAdminMandateIds(db, userId) + + if not adminMandateIds: + provisionResult = rootInterface._provisionMandateForUser( + userId=userId, + mandateType="personal", + mandateName=context.user.fullName or context.user.username, + planKey="TRIAL_7D", + ) + adminMandateIds = [provisionResult["mandateId"]] + logger.info(f"Auto-provisioned personal mandate {adminMandateIds[0]} for user {userId} on Store access") + result = [] for mid in adminMandateIds: records = db.getRecordset(Mandate, recordFilter={"id": mid}) @@ -253,7 +268,7 @@ def activateStoreFeature( ) -> Dict[str, Any]: """ Activate a store feature. Creates a new FeatureInstance in the target mandate. - If mandateId is None and user has no admin mandate, auto-creates a personal mandate. + If user has no admin mandate, auto-creates a personal mandate. """ featureCode = data.featureCode userId = str(context.user.id) @@ -269,27 +284,6 @@ def activateStoreFeature( mandateId = data.mandateId - # Auto-create personal mandate if user has no admin mandates - if not mandateId: - adminMandateIds = _getUserAdminMandateIds(db, userId) - if not adminMandateIds: - provisionResult = rootInterface._provisionMandateForUser( - userId=userId, - mandateType="personal", - mandateName=context.user.fullName or context.user.username, - planKey="TRIAL_7D", - ) - mandateId = provisionResult["mandateId"] - logger.info(f"Auto-created personal mandate {mandateId} for user {userId} via store") - elif len(adminMandateIds) == 1: - mandateId = adminMandateIds[0] - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="mandateId is required when user has multiple admin mandates" - ) - - # Verify user is admin in target mandate if not _isUserAdminInMandate(db, userId, mandateId): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Not admin in target mandate") @@ -323,19 +317,23 @@ def activateStoreFeature( instanceId = instance.get("id") if isinstance(instance, dict) else instance.id - # Grant FeatureAccess with admin role + # Grant FeatureAccess with admin role — MUST be feature-specific (e.g. workspace-admin) instanceRoles = db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) adminRoleId = None for ir in instanceRoles: - if "admin" in (ir.get("roleLabel") or "").lower(): + roleLabel = (ir.get("roleLabel") or "").lower() + if roleLabel.endswith("-admin"): adminRoleId = ir.get("id") break - fa = FeatureAccess(userId=userId, featureInstanceId=instanceId, enabled=True) - createdFa = db.recordCreate(FeatureAccess, fa.model_dump()) - if adminRoleId and createdFa: - far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminRoleId) - db.recordCreate(FeatureAccessRole, far.model_dump()) + if not adminRoleId: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"No feature-specific admin role (e.g. {featureCode}-admin) found for instance {instanceId}. " + f"Template roles were not correctly copied.", + ) + + rootInterface.createFeatureAccess(userId, instanceId, roleIds=[adminRoleId]) # Sync subscription quantity try: diff --git a/modules/routes/routeSubscription.py b/modules/routes/routeSubscription.py index 8334a8c0..0c5eed4e 100644 --- a/modules/routes/routeSubscription.py +++ b/modules/routes/routeSubscription.py @@ -12,7 +12,7 @@ Endpoints: - POST /api/subscription/force-cancel — sysadmin immediate cancel (by ID) """ -from fastapi import APIRouter, HTTPException, Depends, Request, Query +from fastapi import APIRouter, HTTPException, Depends, Request, Query, Path from fastapi import status from typing import Dict, Any, List, Optional import logging @@ -435,3 +435,59 @@ def getFilterValues( crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) return _extractDistinctValues(crossFiltered, column) + + +# ============================================================ +# Data Volume Usage per Mandate +# ============================================================ + +@router.get("/data-volume/{targetMandateId}") +@limiter.limit("60/minute") +def _getDataVolumeUsage( + request: Request, + targetMandateId: str = Path(..., description="Mandate ID to check volume for"), + context: RequestContext = Depends(getRequestContext), +): + """Calculate current data volume usage for a mandate vs. plan limit.""" + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelFiles import FileItem + from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionPlan + from modules.datamodels.datamodelFeature import FeatureInstance + + rootIf = getRootInterface() + mandateId = targetMandateId + + instances = rootIf.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) + totalBytes = 0 + for inst in instances: + instId = inst.get("id") if isinstance(inst, dict) else getattr(inst, "id", None) + if not instId: + continue + files = rootIf.db.getRecordset(FileItem, recordFilter={"featureInstanceId": instId}) + for f in files: + size = f.get("fileSize") if isinstance(f, dict) else getattr(f, "fileSize", 0) + totalBytes += (size or 0) + + usedMB = round(totalBytes / (1024 * 1024), 2) + + maxMB = None + subs = rootIf.db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId}) + for sub in subs: + planKey = sub.get("planKey") if isinstance(sub, dict) else getattr(sub, "planKey", "") + if planKey: + plans = rootIf.db.getRecordset(SubscriptionPlan, recordFilter={"planKey": planKey}) + for plan in plans: + limit = plan.get("maxDataVolumeMB") if isinstance(plan, dict) else getattr(plan, "maxDataVolumeMB", None) + if limit: + maxMB = limit + break + if maxMB: + break + + return { + "mandateId": mandateId, + "usedMB": usedMB, + "maxDataVolumeMB": maxMB, + "percentUsed": round((usedMB / maxMB) * 100, 1) if maxMB else None, + "warning": usedMB >= (maxMB * 0.8) if maxMB else False, + } diff --git a/modules/routes/routeSystem.py b/modules/routes/routeSystem.py index 60e498bd..5a08202c 100644 --- a/modules/routes/routeSystem.py +++ b/modules/routes/routeSystem.py @@ -247,12 +247,12 @@ def _buildDynamicBlock( # Sort views by order views.sort(key=lambda v: v["order"]) - # Add instance to feature featuresMap[featureKey]["instances"].append({ "id": str(instance.id), "uiLabel": instance.label, "order": 10, - "views": views + "views": views, + "isAdmin": permissions.get("isAdmin", False), }) # Build final structure diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index 37b8b0ba..541835a3 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -177,8 +177,11 @@ class AiService: # Neutralize prompt if enabled (before AI call) _wasNeutralized = False + _excludedDocs: List[str] = [] if self._shouldNeutralize(request): - request, _wasNeutralized = self._neutralizeRequest(request) + request, _wasNeutralized, _excludedDocs = self._neutralizeRequest(request) + if _excludedDocs: + logger.warning(f"Neutralization partial failures (continuing): {_excludedDocs}") # Set billing callback on aiObjects BEFORE the AI call # This callback is invoked by _callWithModel() after EVERY individual model call @@ -199,6 +202,15 @@ class AiService: if _wasNeutralized and response and hasattr(response, 'content') and response.content: response.content = self._rehydrateResponse(response.content) + # Attach neutralization exclusion metadata if any parts failed + if _excludedDocs and response: + if not hasattr(response, 'metadata') or response.metadata is None: + response.metadata = {} + if isinstance(response.metadata, dict): + response.metadata["neutralizationExcluded"] = _excludedDocs + elif hasattr(response.metadata, '__dict__'): + response.metadata.neutralizationExcluded = _excludedDocs + return response async def callAiStream(self, request: AiCallRequest): @@ -217,15 +229,26 @@ class AiService: # Neutralize prompt if enabled (before streaming) _wasNeutralized = False + _excludedDocs: List[str] = [] if self._shouldNeutralize(request): - request, _wasNeutralized = self._neutralizeRequest(request) + request, _wasNeutralized, _excludedDocs = self._neutralizeRequest(request) + if _excludedDocs: + logger.warning(f"Neutralization partial failures in stream (continuing): {_excludedDocs}") self.aiObjects.billingCallback = self._createBillingCallback() try: async for chunk in self.aiObjects.callWithTextContextStream(request): # Rehydrate the final AiCallResponse (non-str chunks are the final response) - if _wasNeutralized and not isinstance(chunk, str) and hasattr(chunk, 'content') and chunk.content: - chunk.content = self._rehydrateResponse(chunk.content) + if not isinstance(chunk, str): + if _wasNeutralized and hasattr(chunk, 'content') and chunk.content: + chunk.content = self._rehydrateResponse(chunk.content) + if _excludedDocs: + if not hasattr(chunk, 'metadata') or chunk.metadata is None: + chunk.metadata = {} + if isinstance(chunk.metadata, dict): + chunk.metadata["neutralizationExcluded"] = _excludedDocs + elif hasattr(chunk.metadata, '__dict__'): + chunk.metadata.neutralizationExcluded = _excludedDocs yield chunk finally: self.aiObjects.billingCallback = None @@ -541,40 +564,71 @@ detectedIntent-Werte: def _shouldNeutralize(self, request: AiCallRequest) -> bool: """Check if this AI request should have neutralization applied. - Only applies to text prompts — not embeddings or image processing.""" + Per-request override: requireNeutralization=True forces it, False skips it. + Only applies to text prompts -- not embeddings or image processing.""" try: + if request.requireNeutralization is False: + return False + if not request.prompt and not request.messages: + return False + if request.requireNeutralization is True: + return True neutralSvc = self._get_service("neutralization") if not neutralSvc: return False config = neutralSvc.getConfig() if hasattr(neutralSvc, 'getConfig') else None if not config or not getattr(config, 'enabled', False): return False - if not request.prompt and not request.messages: - return False return True except Exception: return False - def _neutralizeRequest(self, request: AiCallRequest) -> Tuple[AiCallRequest, bool]: - """Neutralize the prompt text in an AiCallRequest. - Returns (modifiedRequest, wasNeutralized). - Raises RuntimeError if neutralization is required but fails (fail-safe).""" + def _neutralizeRequest(self, request: AiCallRequest) -> Tuple[AiCallRequest, bool, List[str]]: + """Neutralize the prompt text and messages in an AiCallRequest. + Returns (modifiedRequest, wasNeutralized, excludedDocs). + Fail-safe: failing parts are excluded instead of aborting the entire call.""" + excludedDocs: List[str] = [] + neutralSvc = self._get_service("neutralization") if not neutralSvc or not hasattr(neutralSvc, 'processText'): - raise RuntimeError("Neutralization required but neutralization service is unavailable") + logger.warning("Neutralization required but neutralization service is unavailable — continuing without neutralization") + excludedDocs.append("Neutralization service unavailable; prompt sent un-neutralized") + return request, False, excludedDocs + + _wasNeutralized = False if request.prompt: - result = neutralSvc.processText(request.prompt) - if result and result.get("neutralized_text"): - request.prompt = result["neutralized_text"] - logger.debug("Neutralized prompt in AiCallRequest") - return request, True - raise RuntimeError( - "Neutralization required but processText returned no neutralized_text — " - "AI call blocked to protect sensitive data" - ) + try: + result = neutralSvc.processText(request.prompt) + if result and result.get("neutralized_text"): + request.prompt = result["neutralized_text"] + _wasNeutralized = True + logger.debug("Neutralized prompt in AiCallRequest") + else: + logger.warning("Neutralization of prompt returned no neutralized_text — sending original prompt") + excludedDocs.append("Prompt neutralization failed; original prompt used") + except Exception as e: + logger.warning(f"Neutralization of prompt failed: {e} — sending original prompt") + excludedDocs.append(f"Prompt neutralization error: {e}") - return request, False + if request.messages and isinstance(request.messages, list): + for idx, msg in enumerate(request.messages): + content = msg.get("content") if isinstance(msg, dict) else None + if not isinstance(content, str) or not content: + continue + try: + result = neutralSvc.processText(content) + if result and result.get("neutralized_text"): + msg["content"] = result["neutralized_text"] + _wasNeutralized = True + else: + logger.warning(f"Neutralization of message[{idx}] returned no neutralized_text — keeping original") + excludedDocs.append(f"Message[{idx}] neutralization failed; original kept") + except Exception as e: + logger.warning(f"Neutralization of message[{idx}] failed: {e} — keeping original") + excludedDocs.append(f"Message[{idx}] neutralization error: {e}") + + return request, _wasNeutralized, excludedDocs def _rehydrateResponse(self, responseText: str) -> str: """Replace neutralization placeholders with original values in AI response.""" diff --git a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py index 77e8530e..14a01557 100644 --- a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py +++ b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py @@ -111,7 +111,7 @@ class KnowledgeService: # 2. Chunk text content objects and create embeddings textObjects = [o for o in contentObjects if o.get("contentType") == "text"] - # Check if file requires neutralization + # Read FileItem attributes for index metadata and neutralization _shouldNeutralize = False try: from modules.datamodels.datamodelFiles import FileItem as _FileItem @@ -119,10 +119,14 @@ class KnowledgeService: _fileRecords = _dbComponent.getRecordset(_FileItem, recordFilter={"id": fileId}) if _dbComponent else [] if _fileRecords: _fileRecord = _fileRecords[0] - _shouldNeutralize = ( - _fileRecord.get("neutralize", False) if isinstance(_fileRecord, dict) - else getattr(_fileRecord, "neutralize", False) - ) + _get = (lambda k, d=None: _fileRecord.get(k, d)) if isinstance(_fileRecord, dict) else (lambda k, d=None: getattr(_fileRecord, k, d)) + _shouldNeutralize = bool(_get("neutralize", False)) + _fileScope = _get("scope") + if _fileScope: + index.scope = _fileScope + _fileCreatedBy = _get("_createdBy") + if _fileCreatedBy: + index.userId = str(_fileCreatedBy) except Exception: pass @@ -201,6 +205,7 @@ class KnowledgeService: if _shouldNeutralize: try: index.neutralizationStatus = "completed" + index.isNeutralized = True self._knowledgeDb.upsertFileContentIndex(index) except Exception as e: logger.debug(f"Could not set neutralizationStatus for file {fileId}: {e}") diff --git a/modules/shared/attributeUtils.py b/modules/shared/attributeUtils.py index 6a857d85..863d7f36 100644 --- a/modules/shared/attributeUtils.py +++ b/modules/shared/attributeUtils.py @@ -258,27 +258,6 @@ def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguag attributes.append(attr_def) - # Append system timestamp fields (set automatically by DatabaseConnector) - systemTimestampFields = [ - ("_createdAt", {"en": "Created at", "de": "Erstellt am", "fr": "Créé le"}), - ("_modifiedAt", {"en": "Modified at", "de": "Geändert am", "fr": "Modifié le"}), - ] - for sysName, sysLabels in systemTimestampFields: - attributes.append({ - "name": sysName, - "type": "timestamp", - "required": False, - "description": "", - "label": sysLabels.get(userLanguage, sysLabels["en"]), - "placeholder": "", - "editable": False, - "visible": True, - "order": len(attributes), - "readonly": True, - "options": None, - "default": None, - }) - return {"model": model_label, "attributes": attributes}