PowerOn/gateway
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

unified data - step 1

Browse source
This commit is contained in:
ValueOn AG 2026-03-24 14:16:46 +01:00
parent f796ae3807
commit 8b161ed410
27 changed files with 1764 additions and 416 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
modules/datamodels/datamodelDataSource.py
View file

@ -30,6 +30,21 @@ class DataSource(BaseModel):
autoSync: bool = Field(default=False, description="Automatically sync on schedule")
lastSynced: Optional[float] = Field(default=None, description="Last sync timestamp")
createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp")
scope: str = Field(
default="personal",
description="Data visibility scope: personal, featureInstance, mandate, global",
json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [
{"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}},
{"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}},
{"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}},
{"value": "global", "label": {"en": "Global", "de": "Global"}},
]}
)
neutralize: bool = Field(
default=False,
description="Whether this data source should be neutralized before AI processing",
json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}
)
registerModelLabels(
@ -48,6 +63,8 @@ registerModelLabels(
"autoSync": {"en": "Auto Sync", "de": "Auto-Sync", "fr": "Synchro auto"},
"lastSynced": {"en": "Last Synced", "de": "Letzter Sync", "fr": "Dernier sync"},
"createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"},
"scope": {"en": "Scope", "de": "Sichtbarkeit"},
"neutralize": {"en": "Neutralize", "de": "Neutralisieren"},
},
)

17
modules/datamodels/datamodelFiles.py
View file

@ -24,6 +24,21 @@ class FileItem(BaseModel):
folderId: Optional[str] = Field(default=None, description="ID of the parent folder", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False})
description: Optional[str] = Field(default=None, description="User-provided description of the file", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False})
status: Optional[str] = Field(default=None, description="Processing status: pending, extracted, embedding, indexed, failed", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
scope: str = Field(
default="personal",
description="Data visibility scope: personal, featureInstance, mandate, global",
json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [
{"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}},
{"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}},
{"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}},
{"value": "global", "label": {"en": "Global", "de": "Global"}},
]}
)
neutralize: bool = Field(
default=False,
description="Whether this file should be neutralized before AI processing",
json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}
)
registerModelLabels(
"FileItem",
@ -41,6 +56,8 @@ registerModelLabels(
"folderId": {"en": "Folder ID", "fr": "ID du dossier"},
"description": {"en": "Description", "fr": "Description"},
"status": {"en": "Status", "fr": "Statut"},
"scope": {"en": "Scope", "de": "Sichtbarkeit"},
"neutralize": {"en": "Neutralize", "de": "Neutralisieren"},
},
)

10
modules/datamodels/datamodelKnowledge.py
View file

@ -34,6 +34,14 @@ class FileContentIndex(BaseModel):
objectSummary: List[Dict[str, Any]] = Field(default_factory=list, description="Compact summary per content object")
extractedAt: float = Field(default_factory=getUtcTimestamp, description="Extraction timestamp")
status: str = Field(default="pending", description="Processing status: pending, extracted, embedding, indexed, failed")
scope: str = Field(
default="personal",
description="Data visibility scope: personal, featureInstance, mandate, global",
)
neutralizationStatus: Optional[str] = Field(
default=None,
description="Neutralization status: completed, failed, skipped, None = not required",
)
registerModelLabels(
@ -54,6 +62,8 @@ registerModelLabels(
"objectSummary": {"en": "Object Summary", "fr": "Résumé des objets"},
"extractedAt": {"en": "Extracted At", "fr": "Extrait le"},
"status": {"en": "Status", "fr": "Statut"},
"scope": {"en": "Scope", "de": "Sichtbarkeit"},
"neutralizationStatus": {"en": "Neutralization Status", "de": "Neutralisierungsstatus"},
},
)

6
modules/datamodels/datamodelSubscription.py
View file

@ -70,6 +70,7 @@ class SubscriptionPlan(BaseModel):
maxUsers: Optional[int] = Field(None, description="Hard cap on active users (None = unlimited)")
maxFeatureInstances: Optional[int] = Field(None, description="Hard cap on active feature instances (None = unlimited)")
trialDays: Optional[int] = Field(None, description="Trial duration in days (only for trial plans)")
maxDataVolumeMB: Optional[int] = Field(None, description="Soft-limit for data volume in MB per mandate (None = unlimited)")
successorPlanKey: Optional[str] = Field(None, description="Plan to transition to when trial ends")
@ -84,6 +85,7 @@ registerModelLabels(
"pricePerFeatureInstanceCHF": {"en": "Price per Instance (CHF)", "de": "Preis pro Instanz (CHF)"},
"maxUsers": {"en": "Max Users", "de": "Max. Benutzer", "fr": "Max. utilisateurs"},
"maxFeatureInstances": {"en": "Max Instances", "de": "Max. Instanzen", "fr": "Max. instances"},
"maxDataVolumeMB": {"en": "Data Volume (MB)", "de": "Datenvolumen (MB)"},
},
)
@ -182,6 +184,7 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = {
autoRenew=False,
maxUsers=None,
maxFeatureInstances=None,
maxDataVolumeMB=None,
),
"TRIAL_7D": SubscriptionPlan(
planKey="TRIAL_7D",
@ -196,6 +199,7 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = {
maxUsers=1,
maxFeatureInstances=3,
trialDays=7,
maxDataVolumeMB=500,
successorPlanKey="STANDARD_MONTHLY",
),
"STANDARD_MONTHLY": SubscriptionPlan(
@ -209,6 +213,7 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = {
billingPeriod=BillingPeriodEnum.MONTHLY,
pricePerUserCHF=90.0,
pricePerFeatureInstanceCHF=150.0,
maxDataVolumeMB=10240,
),
"STANDARD_YEARLY": SubscriptionPlan(
planKey="STANDARD_YEARLY",
@ -221,6 +226,7 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = {
billingPeriod=BillingPeriodEnum.YEARLY,
pricePerUserCHF=1080.0,
pricePerFeatureInstanceCHF=1800.0,
maxDataVolumeMB=10240,
),
}

60
modules/datamodels/datamodelUam.py
View file

@ -10,7 +10,7 @@ Multi-Tenant Design:
"""
import uuid
from typing import Optional, List
from typing import Optional, List, Dict
from enum import Enum
from pydantic import BaseModel, Field, EmailStr, field_validator, computed_field
from modules.shared.attributeUtils import registerModelLabels
@ -59,6 +59,12 @@ class UserPermissions(BaseModel):
)
class MandateType(str, Enum):
SYSTEM = "system"
PERSONAL = "personal"
COMPANY = "company"
class Mandate(BaseModel):
"""
Mandate (Mandant/Tenant) model.
@ -88,6 +94,15 @@ class Mandate(BaseModel):
description="Whether this is a system mandate (e.g. root mandate). Cannot be deleted.",
json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": False}
)
mandateType: MandateType = Field(
default=MandateType.COMPANY,
description="Fachlicher Mandantentyp: system (Root), personal (Solo), company (Team). Mutabel, rein informativ — keine Feature-Gates.",
json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [
{"value": "system", "label": {"en": "System", "de": "System"}},
{"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}},
{"value": "company", "label": {"en": "Company", "de": "Unternehmen"}},
]}
)
@field_validator('isSystem', mode='before')
@classmethod
@ -97,6 +112,18 @@ class Mandate(BaseModel):
return False
return v
@field_validator('mandateType', mode='before')
@classmethod
def _coerceMandateType(cls, v):
if v is None:
return MandateType.COMPANY
if isinstance(v, str):
try:
return MandateType(v)
except ValueError:
return MandateType.COMPANY
return v
registerModelLabels(
"Mandate",
@ -107,6 +134,7 @@ registerModelLabels(
"label": {"en": "Label", "de": "Label", "fr": "Libellé"},
"enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"},
"isSystem": {"en": "System Mandate", "de": "System-Mandant", "fr": "Mandat système"},
"mandateType": {"en": "Mandate Type", "de": "Mandantentyp", "fr": "Type de mandat"},
},
)
@ -289,3 +317,33 @@ registerModelLabels(
"resetTokenExpires": {"en": "Reset Token Expires", "de": "Token läuft ab", "fr": "Expiration du jeton"},
},
)
class UserVoicePreferences(BaseModel):
"""User-level voice/language preferences, shared across all features."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key")
userId: str = Field(description="User ID")
mandateId: Optional[str] = Field(default=None, description="Mandate scope (None = global for user)")
sttLanguage: str = Field(default="de-DE", description="Speech-to-text language code")
ttsLanguage: str = Field(default="de-DE", description="Text-to-speech language code")
ttsVoice: Optional[str] = Field(default=None, description="Preferred TTS voice identifier")
ttsVoiceMap: Optional[Dict[str, str]] = Field(default=None, description="Language-to-voice mapping")
translationSourceLanguage: Optional[str] = Field(default=None, description="Source language for translations")
translationTargetLanguage: Optional[str] = Field(default=None, description="Target language for translations")
registerModelLabels(
"UserVoicePreferences",
{"en": "Voice Preferences", "de": "Spracheinstellungen", "fr": "Préférences vocales"},
{
"id": {"en": "ID", "de": "ID", "fr": "ID"},
"userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"},
"mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"},
"sttLanguage": {"en": "STT Language", "de": "STT-Sprache", "fr": "Langue STT"},
"ttsLanguage": {"en": "TTS Language", "de": "TTS-Sprache", "fr": "Langue TTS"},
"ttsVoice": {"en": "TTS Voice", "de": "TTS-Stimme", "fr": "Voix TTS"},
"ttsVoiceMap": {"en": "Voice Map", "de": "Stimmen-Zuordnung", "fr": "Carte des voix"},
"translationSourceLanguage": {"en": "Translation Source", "de": "Übersetzung Quelle", "fr": "Langue source"},
"translationTargetLanguage": {"en": "Translation Target", "de": "Übersetzung Ziel", "fr": "Langue cible"},
},
)

6
modules/datamodels/datamodelVoice.py
View file

@ -1,7 +1,7 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""Voice settings datamodel — re-exported from workspace feature for backward compatibility."""
"""Voice settings datamodel — re-exported from UAM for central voice preferences."""
from modules.features.workspace.datamodelFeatureWorkspace import VoiceSettings
from modules.datamodels.datamodelUam import UserVoicePreferences
__all__ = ["VoiceSettings"]
__all__ = ["UserVoicePreferences"]

17
modules/features/neutralization/datamodelFeatureNeutralizer.py
View file

@ -3,17 +3,32 @@
"""Neutralizer models: DataNeutraliserConfig and DataNeutralizerAttributes."""
import uuid
from enum import Enum
from typing import Optional
from pydantic import BaseModel, Field
from modules.shared.attributeUtils import registerModelLabels
class DataScope(str, Enum):
PERSONAL = "personal"
FEATURE_INSTANCE = "featureInstance"
MANDATE = "mandate"
GLOBAL = "global"
class DataNeutraliserConfig(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
mandateId: str = Field(description="ID of the mandate this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
featureInstanceId: str = Field(description="ID of the feature instance this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
userId: str = Field(description="ID of the user who created this configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
enabled: bool = Field(default=True, description="Whether data neutralization is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False})
scope: str = Field(default="personal", description="Data visibility scope: personal, featureInstance, mandate, global", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [
{"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}},
{"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}},
{"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}},
{"value": "global", "label": {"en": "Global", "de": "Global"}},
]})
neutralizationStatus: str = Field(default="not_required", description="Status of neutralization: pending, completed, failed, not_required", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
namesToParse: str = Field(default="", description="Multiline list of names to parse for neutralization", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False})
sharepointSourcePath: str = Field(default="", description="SharePoint path to read files for neutralization", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False})
sharepointTargetPath: str = Field(default="", description="SharePoint path to store neutralized files", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False})
@ -26,6 +41,8 @@ registerModelLabels(
"featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"},
"userId": {"en": "User ID", "fr": "ID utilisateur"},
"enabled": {"en": "Enabled", "fr": "Activé"},
"scope": {"en": "Scope", "fr": "Portée"},
"neutralizationStatus": {"en": "Neutralization Status", "fr": "Statut de neutralisation"},
"namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"},
"sharepointSourcePath": {"en": "Source Path", "fr": "Chemin source"},
"sharepointTargetPath": {"en": "Target Path", "fr": "Chemin cible"},

15
modules/features/neutralization/interfaceFeatureNeutralizer.py
View file

@ -212,6 +212,21 @@ class InterfaceFeatureNeutralizer:
logger.error(f"Error getting attribute by ID: {str(e)}")
return None
def deleteAttributeById(self, attributeId: str) -> bool:
"""Delete a single neutralization attribute by its ID"""
try:
attribute = self.getAttributeById(attributeId)
if not attribute:
logger.warning(f"Attribute {attributeId} not found for deletion")
return False
self.db.recordDelete(DataNeutralizerAttributes, attributeId)
logger.info(f"Deleted neutralization attribute {attributeId}")
return True
except Exception as e:
logger.error(f"Error deleting attribute by ID: {str(e)}")
return False
def createAttribute(
self,
attributeId: str,

6
modules/features/neutralization/neutralizePlayground.py
View file

@ -7,6 +7,7 @@ from urllib.parse import urlparse, unquote
from modules.datamodels.datamodelUam import User
from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig
from .interfaceFeatureNeutralizer import getInterface as _getNeutralizerInterface
from modules.serviceHub import getInterface as getServices
logger = logging.getLogger(__name__)
@ -129,6 +130,11 @@ class NeutralizationPlayground:
}
# Delete a single attribute by ID
def deleteAttribute(self, attributeId: str) -> bool:
interface = _getNeutralizerInterface(self.currentUser, self.mandateId, self.featureInstanceId)
return interface.deleteAttributeById(attributeId)
# Cleanup attributes
def cleanAttributes(self, fileId: str) -> bool:
return self.services.neutralization.deleteNeutralizationAttributes(fileId)

60
modules/features/neutralization/routeFeatureNeutralizer.py
View file

@ -317,6 +317,66 @@ def get_neutralization_stats(
detail=f"Error getting neutralization stats: {str(e)}"
)
@router.delete("/attributes/single/{attributeId}", response_model=Dict[str, str])
@limiter.limit("30/minute")
def deleteAttribute(
request: Request,
attributeId: str = Path(..., description="Attribute ID to delete"),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, str]:
"""Delete a single neutralization attribute by ID."""
try:
service = NeutralizationPlayground(
context.user,
str(context.mandateId) if context.mandateId else "",
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
)
success = service.deleteAttribute(attributeId)
if success:
return {"message": f"Attribute {attributeId} deleted"}
else:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Attribute {attributeId} not found"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deleting attribute: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/retrigger", response_model=Dict[str, str])
@limiter.limit("10/minute")
def retriggerNeutralization(
request: Request,
retriggerData: Dict[str, str] = Body(...),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, str]:
"""Re-trigger neutralization for a specific file."""
try:
fileId = retriggerData.get("fileId", "")
if not fileId:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="fileId is required"
)
service = NeutralizationPlayground(
context.user,
str(context.mandateId) if context.mandateId else "",
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
)
service.cleanupFileAttributes(fileId)
return {"message": f"Neutralization re-triggered for file {fileId}", "fileId": fileId}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error re-triggering neutralization: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/attributes/{fileId}", response_model=Dict[str, str])
@limiter.limit("10/minute")
def cleanup_file_attributes(

39
modules/features/workspace/datamodelFeatureWorkspace.py
View file

@ -1,29 +1,13 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""Workspace feature data models — VoiceSettings and WorkspaceUserSettings."""
"""Workspace feature data models — WorkspaceUserSettings."""
from typing import Dict, Any, Optional
from typing import Optional
from pydantic import BaseModel, Field
from modules.shared.attributeUtils import registerModelLabels
from modules.shared.timeUtils import getUtcTimestamp
import uuid
class VoiceSettings(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
userId: str = Field(description="ID of the user these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
mandateId: str = Field(description="ID of the mandate these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
featureInstanceId: str = Field(description="ID of the feature instance these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True})
sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True})
ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True})
ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True})
ttsVoiceMap: Dict[str, Any] = Field(default_factory=dict, description="Per-language voice mapping, e.g. {'de-DE': {'voiceName': 'de-DE-Wavenet-A'}, 'en-US': {'voiceName': 'en-US-Wavenet-C'}}", json_schema_extra={"frontend_type": "json", "frontend_readonly": False, "frontend_required": False})
translationEnabled: bool = Field(default=True, description="Whether translation is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False})
targetLanguage: str = Field(default="en-US", description="Target language for translation", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False})
creationDate: float = Field(default_factory=getUtcTimestamp, description="Date when the settings were created (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False})
lastModified: float = Field(default_factory=getUtcTimestamp, description="Date when the settings were last modified (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False})
class WorkspaceUserSettings(BaseModel):
"""Per-user workspace settings. None values mean 'use instance default'."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
@ -33,25 +17,6 @@ class WorkspaceUserSettings(BaseModel):
maxAgentRounds: Optional[int] = Field(default=None, description="Max agent rounds override (None = instance default)", json_schema_extra={"frontend_type": "number", "frontend_readonly": False, "frontend_required": False})
registerModelLabels(
"VoiceSettings",
{"en": "Voice Settings", "fr": "Paramètres vocaux"},
{
"id": {"en": "ID", "fr": "ID"},
"userId": {"en": "User ID", "fr": "ID utilisateur"},
"mandateId": {"en": "Mandate ID", "fr": "ID du mandat"},
"featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"},
"sttLanguage": {"en": "STT Language", "fr": "Langue STT"},
"ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"},
"ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"},
"ttsVoiceMap": {"en": "TTS Voice Map", "fr": "Carte des voix TTS"},
"translationEnabled": {"en": "Translation Enabled", "fr": "Traduction activée"},
"targetLanguage": {"en": "Target Language", "fr": "Langue cible"},
"creationDate": {"en": "Creation Date", "fr": "Date de création"},
"lastModified": {"en": "Last Modified", "fr": "Dernière modification"},
},
)
registerModelLabels(
"WorkspaceUserSettings",
{"en": "Workspace User Settings", "de": "Workspace Benutzereinstellungen"},

118
modules/features/workspace/interfaceFeatureWorkspace.py
View file

@ -10,7 +10,7 @@ from typing import Dict, Any, Optional
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.datamodels.datamodelUam import User
from modules.features.workspace.datamodelFeatureWorkspace import VoiceSettings, WorkspaceUserSettings
from modules.features.workspace.datamodelFeatureWorkspace import WorkspaceUserSettings
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC
from modules.security.rbac import RbacClass
from modules.shared.configuration import APP_CONFIG
@ -62,122 +62,6 @@ class WorkspaceObjects:
self.featureInstanceId = featureInstanceId
self.db.updateContext(self.userId)
# =========================================================================
# VoiceSettings CRUD
# =========================================================================
def getVoiceSettings(self, userId: Optional[str] = None) -> Optional[VoiceSettings]:
try:
targetUserId = userId or self.userId
if not targetUserId:
logger.error("No user ID provided for voice settings")
return None
recordFilter: Dict[str, Any] = {"userId": targetUserId}
if self.featureInstanceId:
recordFilter["featureInstanceId"] = self.featureInstanceId
filteredSettings = getRecordsetWithRBAC(
self.db, VoiceSettings, self.currentUser,
recordFilter=recordFilter, mandateId=self.mandateId,
)
if not filteredSettings:
return None
settingsData = filteredSettings[0]
if not settingsData.get("creationDate"):
settingsData["creationDate"] = getUtcTimestamp()
if not settingsData.get("lastModified"):
settingsData["lastModified"] = getUtcTimestamp()
return VoiceSettings(**settingsData)
except Exception as e:
logger.error(f"Error getting voice settings: {e}")
return None
def createVoiceSettings(self, settingsData: Dict[str, Any]) -> Dict[str, Any]:
try:
if "userId" not in settingsData:
settingsData["userId"] = self.userId
if "mandateId" not in settingsData:
settingsData["mandateId"] = self.mandateId
if "featureInstanceId" not in settingsData:
settingsData["featureInstanceId"] = self.featureInstanceId
existing = self.getVoiceSettings(settingsData["userId"])
if existing:
raise ValueError(f"Voice settings already exist for user {settingsData['userId']}")
createdRecord = self.db.recordCreate(VoiceSettings, settingsData)
if not createdRecord or not createdRecord.get("id"):
raise ValueError("Failed to create voice settings record")
logger.info(f"Created voice settings for user {settingsData['userId']}")
return createdRecord
except Exception as e:
logger.error(f"Error creating voice settings: {e}")
raise
def updateVoiceSettings(self, userId: str, updateData: Dict[str, Any]) -> Dict[str, Any]:
try:
existing = self.getVoiceSettings(userId)
if not existing:
raise ValueError(f"Voice settings not found for user {userId}")
updateData["lastModified"] = getUtcTimestamp()
success = self.db.recordModify(VoiceSettings, existing.id, updateData)
if not success:
raise ValueError("Failed to update voice settings record")
updated = self.getVoiceSettings(userId)
if not updated:
raise ValueError("Failed to retrieve updated voice settings")
logger.info(f"Updated voice settings for user {userId}")
return updated.model_dump()
except Exception as e:
logger.error(f"Error updating voice settings: {e}")
raise
def deleteVoiceSettings(self, userId: str) -> bool:
try:
existing = self.getVoiceSettings(userId)
if not existing:
return False
success = self.db.recordDelete(VoiceSettings, existing.id)
if success:
logger.info(f"Deleted voice settings for user {userId}")
return success
except Exception as e:
logger.error(f"Error deleting voice settings: {e}")
return False
def getOrCreateVoiceSettings(self, userId: Optional[str] = None) -> VoiceSettings:
targetUserId = userId or self.userId
if not targetUserId:
raise ValueError("No user ID provided for voice settings")
existing = self.getVoiceSettings(targetUserId)
if existing:
return existing
defaultSettings = {
"userId": targetUserId,
"mandateId": self.mandateId,
"featureInstanceId": self.featureInstanceId,
"sttLanguage": "de-DE",
"ttsLanguage": "de-DE",
"ttsVoice": "de-DE-KatjaNeural",
"translationEnabled": True,
"targetLanguage": "en-US",
}
createdRecord = self.createVoiceSettings(defaultSettings)
return VoiceSettings(**createdRecord)
# =========================================================================
# WorkspaceUserSettings CRUD
# =========================================================================

22
modules/interfaces/interfaceBootstrap.py
View file

@ -92,9 +92,25 @@ def initBootstrap(db: DatabaseConnector) -> None:
# Seed automation templates (after admin user exists)
initAutomationTemplates(db, adminUserId)
# Initialize feature instances for root mandate
# Run root-user migration (one-time, sets completion flag)
migrationDone = False
try:
from modules.migration.migrateRootUsers import migrateRootUsers, _isMigrationCompleted
migrationDone = _isMigrationCompleted(db)
if not migrationDone:
# Create root instances first (needed for migration), then migrate
if mandateId:
initRootMandateFeatures(db, mandateId)
result = migrateRootUsers(db)
migrationDone = result.get("status") != "error"
else:
migrationDone = True
except Exception as e:
logger.error(f"Root user migration failed: {e}")
# After migration: root mandate is purely technical — no feature instances
if not migrationDone and mandateId:
initRootMandateFeatures(db, mandateId)
# Remove feature instances for features that no longer exist in the codebase
_cleanupRemovedFeatureInstances(db)
@ -310,6 +326,8 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]:
if existingMandates:
mandateId = existingMandates[0].get("id")
logger.info(f"Root mandate already exists with ID {mandateId}")
# Ensure mandateType is set to system
db.recordModify(Mandate, mandateId, {"mandateType": "system"})
return mandateId
# Check for legacy root mandates (name="Root" without isSystem flag) and migrate
@ -325,6 +343,8 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]:
createdMandate = db.recordCreate(Mandate, rootMandate)
mandateId = createdMandate.get("id")
logger.info(f"Root mandate created with ID {mandateId}")
# mandateType already set via Mandate constructor, but ensure:
db.recordModify(Mandate, mandateId, {"mandateType": "system"})
return mandateId

225
modules/interfaces/interfaceDbApp.py
View file

@ -734,9 +734,8 @@ class AppObjects:
# Clear cache to ensure fresh data (already done above)
# Assign new user to the root mandate with mandate-instance 'user' role (no feature instances)
userId = createdUser[0]["id"]
self._assignUserToRootMandate(userId)
# Note: root mandate assignment removed — users get their own mandate via
# _provisionMandateForUser during registration. Root mandate is purely technical.
return User(**createdUser[0])
@ -1456,6 +1455,163 @@ class AppObjects:
return Mandate(**createdRecord)
def _provisionMandateForUser(self, userId: str, mandateType: str, mandateName: str, planKey: str) -> Dict[str, Any]:
"""
Atomic provisioning: create Mandate + UserMandate + Subscription + auto-create FeatureInstances.
Internal method — bypasses RBAC (used during registration when user has no permissions yet).
"""
from modules.datamodels.datamodelUam import MandateType
from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
from modules.interfaces.interfaceFeatures import getFeatureInterface
from modules.system.registry import loadFeatureMainModules
plan = BUILTIN_PLANS.get(planKey)
if not plan:
raise ValueError(f"Unknown plan: {planKey}")
mandateData = Mandate(
name=mandateName,
label=mandateName,
enabled=True,
isSystem=False,
mandateType=MandateType(mandateType),
)
createdMandate = self.db.recordCreate(Mandate, mandateData)
if not createdMandate or not createdMandate.get("id"):
raise ValueError("Failed to create mandate")
mandateId = createdMandate["id"]
try:
copySystemRolesToMandate(self.db, mandateId)
adminRoleId = None
mandateRoles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId, "featureInstanceId": None})
for r in mandateRoles:
if "admin" in (r.get("roleLabel") or "").lower():
adminRoleId = r.get("id")
break
userMandate = UserMandate(userId=userId, mandateId=mandateId, enabled=True)
createdUm = self.db.recordCreate(UserMandate, userMandate.model_dump())
if adminRoleId and createdUm:
umRole = UserMandateRole(userMandateId=createdUm["id"], roleId=adminRoleId)
self.db.recordCreate(UserMandateRole, umRole.model_dump())
subscription = MandateSubscription(
mandateId=mandateId,
planKey=planKey,
status=SubscriptionStatusEnum.PENDING,
)
if plan.trialDays:
pass # trialEndsAt set on ACTIVE transition
self.db.recordCreate(MandateSubscription, subscription.model_dump())
featureInterface = getFeatureInterface(self.db)
mainModules = loadFeatureMainModules()
createdInstances = []
for featureName, module in mainModules.items():
if not hasattr(module, "getFeatureDefinition"):
continue
try:
featureDef = module.getFeatureDefinition()
if not featureDef.get("autoCreateInstance", False):
continue
featureCode = featureDef.get("code", featureName)
featureLabel = featureDef.get("label", {}).get("en", featureName)
instance = featureInterface.createFeatureInstance(
featureCode=featureCode,
mandateId=mandateId,
label=featureLabel,
enabled=True,
copyTemplateRoles=True,
)
if instance:
instanceId = instance.get("id") if isinstance(instance, dict) else instance.id
createdInstances.append(instanceId)
instanceRoles = self.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId})
adminInstRoleId = None
for ir in instanceRoles:
if "admin" in (ir.get("roleLabel") or "").lower():
adminInstRoleId = ir.get("id")
break
fa = FeatureAccess(userId=userId, featureInstanceId=instanceId, enabled=True)
createdFa = self.db.recordCreate(FeatureAccess, fa.model_dump())
if adminInstRoleId and createdFa:
far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminInstRoleId)
self.db.recordCreate(FeatureAccessRole, far.model_dump())
except Exception as e:
logger.error(f"Error auto-creating instance for '{featureName}': {e}")
logger.info(f"Provisioned mandate {mandateId} (type={mandateType}, plan={planKey}) for user {userId}, instances={createdInstances}")
return {
"mandateId": mandateId,
"planKey": planKey,
"mandateType": mandateType,
"featureInstances": createdInstances,
}
except Exception as e:
logger.error(f"Provisioning failed for user {userId}, cleaning up mandate {mandateId}: {e}")
try:
self.db.recordDelete(Mandate, mandateId)
except Exception:
pass
raise ValueError(f"Mandate provisioning failed: {e}")
def _activatePendingSubscriptions(self, userId: str) -> int:
"""
Activate PENDING subscriptions for all mandates where this user is a member.
Called on login — trial period begins NOW, not at registration.
Returns number of activated subscriptions.
"""
from modules.datamodels.datamodelSubscription import (
MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS,
)
from datetime import datetime, timezone, timedelta
activated = 0
userMandates = self.db.getRecordset(
UserMandate, recordFilter={"userId": userId, "enabled": True}
)
for um in userMandates:
mandateId = um.get("mandateId")
subs = self.db.getRecordset(
MandateSubscription,
recordFilter={"mandateId": mandateId, "status": SubscriptionStatusEnum.PENDING.value}
)
for sub in subs:
subId = sub.get("id")
planKey = sub.get("planKey")
plan = BUILTIN_PLANS.get(planKey)
now = datetime.now(timezone.utc)
updateData = {
"status": SubscriptionStatusEnum.TRIALING.value if plan and plan.trialDays else SubscriptionStatusEnum.ACTIVE.value,
"currentPeriodStart": now.isoformat(),
}
if plan and plan.trialDays:
trialEnd = now + timedelta(days=plan.trialDays)
updateData["trialEndsAt"] = trialEnd.isoformat()
updateData["currentPeriodEnd"] = trialEnd.isoformat()
elif plan and plan.billingPeriod:
from modules.datamodels.datamodelSubscription import BillingPeriodEnum
if plan.billingPeriod == BillingPeriodEnum.MONTHLY:
updateData["currentPeriodEnd"] = (now + timedelta(days=30)).isoformat()
elif plan.billingPeriod == BillingPeriodEnum.YEARLY:
updateData["currentPeriodEnd"] = (now + timedelta(days=365)).isoformat()
try:
self.db.recordModify(MandateSubscription, subId, updateData)
activated += 1
logger.info(f"Activated subscription {subId} (plan={planKey}) for mandate {mandateId}: {updateData.get('status')}")
except Exception as e:
logger.error(f"Failed to activate subscription {subId}: {e}")
return activated
def updateMandate(self, mandateId: str, updateData: Dict[str, Any]) -> Mandate:
"""Updates a mandate if user has access."""
try:
@ -1493,33 +1649,68 @@ class AppObjects:
logger.error(f"Error updating mandate: {str(e)}")
raise ValueError(f"Failed to update mandate: {str(e)}")
def deleteMandate(self, mandateId: str) -> bool:
"""Deletes a mandate if user has access. System mandates cannot be deleted."""
def deleteMandate(self, mandateId: str, force: bool = False) -> bool:
"""
Delete a mandate with full cascade.
Default (force=False): soft-delete — sets enabled=false.
With force=True: hard-delete — removes all related data.
System mandates (isSystem=True) cannot be deleted.
"""
try:
# Check if mandate exists and user has access
mandate = self.getMandate(mandateId)
if not mandate:
return False
# System mandates (isSystem=True) cannot be deleted
if getattr(mandate, "isSystem", False):
raise ValueError(f"System mandate '{mandate.name}' cannot be deleted")
if not self.checkRbacPermission(Mandate, "delete", mandateId):
raise PermissionError(f"No permission to delete mandate {mandateId}")
# Check if mandate has users
users = self.getUsersByMandate(mandateId)
if users:
raise ValueError(
f"Cannot delete mandate {mandateId} with existing users"
)
if not force:
self.db.recordModify(Mandate, mandateId, {"enabled": False})
logger.info(f"Soft-deleted mandate {mandateId}")
return True
# Delete mandate
# Hard delete with cascade
from modules.datamodels.datamodelSubscription import MandateSubscription
# 1. Delete FeatureAccess + FeatureAccessRole for all instances in this mandate
instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
for inst in instances:
instId = inst.get("id")
accesses = self.db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instId})
for access in accesses:
self.db.recordDelete(FeatureAccess, access.get("id"))
self.db.recordDelete(FeatureInstance, instId)
logger.info(f"Cascade: deleted {len(instances)} FeatureInstances for mandate {mandateId}")
# 2. Delete UserMandate + UserMandateRole
memberships = self.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId})
for um in memberships:
self.db.recordDelete(UserMandate, um.get("id"))
logger.info(f"Cascade: deleted {len(memberships)} UserMandates for mandate {mandateId}")
# 3. Delete MandateSubscriptions
subs = self.db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId})
for sub in subs:
self.db.recordDelete(MandateSubscription, sub.get("id"))
logger.info(f"Cascade: deleted {len(subs)} subscriptions for mandate {mandateId}")
# 4. Delete mandate-level Roles
from modules.datamodels.datamodelRbac import Role, AccessRule
roles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId})
for role in roles:
rules = self.db.getRecordset(AccessRule, recordFilter={"roleId": role.get("id")})
for rule in rules:
self.db.recordDelete(AccessRule, rule.get("id"))
self.db.recordDelete(Role, role.get("id"))
logger.info(f"Cascade: deleted {len(roles)} Roles for mandate {mandateId}")
# 5. Delete mandate record
success = self.db.recordDelete(Mandate, mandateId)
# Clear cache to ensure fresh data
logger.info(f"Hard-deleted mandate {mandateId}")
return success
except Exception as e:

97
modules/interfaces/interfaceDbKnowledge.py
View file

@ -29,6 +29,7 @@ class KnowledgeObjects:
def __init__(self):
self.currentUser: Optional[User] = None
self.userId: Optional[str] = None
self._scopeCache: Dict[str, List[str]] = {}
self._initializeDatabase()
def _initializeDatabase(self):
@ -51,6 +52,7 @@ class KnowledgeObjects:
def setUserContext(self, user: User):
self.currentUser = user
self.userId = user.id if user else None
self._scopeCache = {}
if self.userId:
self.db.updateContext(self.userId)
@ -215,6 +217,67 @@ class KnowledgeObjects:
# Semantic Search
# =========================================================================
def _buildScopeFilter(self, userId: str = None, featureInstanceId: str = None, mandateId: str = None) -> dict:
"""Build a scope-aware filter for RAG queries.
Returns a filter dict that includes records visible to this user context."""
return {
"userId": userId,
"featureInstanceId": featureInstanceId,
"mandateId": mandateId,
}
def _getScopedFileIds(self, userId: str = None, featureInstanceId: str = None, mandateId: str = None, isSysAdmin: bool = False) -> List[str]:
"""Collect FileContentIndex IDs visible under the scope union:
- scope=personal AND userId matches
- scope=featureInstance AND featureInstanceId matches
- scope=mandate AND mandateId matches
- scope=global (only if isSysAdmin)
"""
_cacheKey = f"{userId}:{featureInstanceId}:{mandateId}:{isSysAdmin}"
if _cacheKey in self._scopeCache:
return self._scopeCache[_cacheKey]
allIds: set = set()
if isSysAdmin:
globalIndexes = self.db.getRecordset(
FileContentIndex, recordFilter={"scope": "global"}
)
for idx in globalIndexes:
fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
if fid:
allIds.add(fid)
if userId:
personalIndexes = self.db.getRecordset(
FileContentIndex, recordFilter={"scope": "personal", "userId": userId}
)
for idx in personalIndexes:
fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
if fid:
allIds.add(fid)
if featureInstanceId:
instanceIndexes = self.db.getRecordset(
FileContentIndex, recordFilter={"scope": "featureInstance", "featureInstanceId": featureInstanceId}
)
for idx in instanceIndexes:
fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
if fid:
allIds.add(fid)
if mandateId:
mandateIndexes = self.db.getRecordset(
FileContentIndex, recordFilter={"scope": "mandate", "mandateId": mandateId}
)
for idx in mandateIndexes:
fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
if fid:
allIds.add(fid)
self._scopeCache[_cacheKey] = list(allIds)
return self._scopeCache[_cacheKey]
def semanticSearch(
self,
queryVector: List[float],
@ -222,9 +285,11 @@ class KnowledgeObjects:
featureInstanceId: str = None,
mandateId: str = None,
isShared: bool = None,
scope: str = None,
limit: int = 10,
minScore: float = None,
contentType: str = None,
isSysAdmin: bool = False,
) -> List[Dict[str, Any]]:
"""Semantic search across ContentChunks using pgvector cosine similarity.
@ -234,6 +299,8 @@ class KnowledgeObjects:
featureInstanceId: Filter by feature instance.
mandateId: Filter by mandate (for Shared Layer lookups).
isShared: If True, search Shared Layer via FileContentIndex join.
scope: If provided, filter by this specific scope value.
If not provided, use scope-union approach (personal + featureInstance + mandate + global).
limit: Max results.
minScore: Minimum cosine similarity (0.0 - 1.0).
contentType: Filter by content type (text, image, etc.).
@ -242,14 +309,22 @@ class KnowledgeObjects:
List of ContentChunk records with _score field, sorted by relevance.
"""
recordFilter = {}
if userId:
recordFilter["userId"] = userId
if featureInstanceId:
recordFilter["featureInstanceId"] = featureInstanceId
if contentType:
recordFilter["contentType"] = contentType
if isShared and mandateId:
if scope:
scopedFileIds = self.db.getRecordset(
FileContentIndex, recordFilter={"scope": scope}
)
fileIds = [
idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
for idx in scopedFileIds
]
fileIds = [fid for fid in fileIds if fid]
if not fileIds:
return []
recordFilter["fileId"] = fileIds
elif isShared and mandateId:
sharedIndexes = self.db.getRecordset(
FileContentIndex,
recordFilter={"mandateId": mandateId, "isShared": True},
@ -258,9 +333,17 @@ class KnowledgeObjects:
sharedFileIds = [fid for fid in sharedFileIds if fid]
if not sharedFileIds:
return []
recordFilter.pop("userId", None)
recordFilter.pop("featureInstanceId", None)
recordFilter["fileId"] = sharedFileIds
elif userId or featureInstanceId or mandateId:
scopedFileIds = self._getScopedFileIds(
userId=userId,
featureInstanceId=featureInstanceId,
mandateId=mandateId,
isSysAdmin=isSysAdmin,
)
if not scopedFileIds:
return []
recordFilter["fileId"] = scopedFileIds
return self.db.semanticSearch(
modelClass=ContentChunk,

1
modules/migration/__init__.py Normal file
View file

@ -0,0 +1 @@
# Migration modules

213
modules/migration/migrateRootUsers.py Normal file
View file

@ -0,0 +1,213 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Migration: Root-Mandant bereinigen.
Moves all end-user data from Root mandate shared instances to own mandates.
Called once from bootstrap, sets a DB flag to prevent re-execution.
"""
import logging
from typing import Optional
logger = logging.getLogger(__name__)
_MIGRATION_FLAG_KEY = "migration_root_users_completed"
def _isMigrationCompleted(db) -> bool:
"""Check if migration has already been executed."""
try:
from modules.datamodels.datamodelUam import Mandate
records = db.getRecordset(Mandate, recordFilter={"name": _MIGRATION_FLAG_KEY})
return len(records) > 0
except Exception:
return False
def _setMigrationCompleted(db) -> None:
"""Set flag that migration is completed (uses a settings-like record)."""
if _isMigrationCompleted(db):
return
try:
from modules.datamodels.datamodelUam import Mandate
flag = Mandate(name=_MIGRATION_FLAG_KEY, label="Migration completed", enabled=False, isSystem=True)
db.recordCreate(Mandate, flag)
logger.info("Migration flag set: root user migration completed")
except Exception as e:
logger.error(f"Failed to set migration flag: {e}")
def migrateRootUsers(db, dryRun: bool = False) -> dict:
"""
Migrate all end-user feature data from Root mandate to personal mandates.
Algorithm:
STEP 1: For each user with FeatureAccess on Root instances:
- If user has own mandate: target = existing mandate
- If not: create personal mandate via _provisionMandateForUser
- For each FeatureAccess: create new instance in target, migrate data, transfer access
STEP 2: Clean up Root:
- Delete all FeatureInstances in Root
- Remove UserMandate for non-sysadmin users
Args:
db: Database connector
dryRun: If True, log actions without making changes
Returns:
Summary dict with migration statistics
"""
if _isMigrationCompleted(db):
logger.info("Root user migration already completed, skipping")
return {"status": "already_completed"}
from modules.datamodels.datamodelUam import Mandate, User, UserInDB
from modules.datamodels.datamodelMembership import (
UserMandate, UserMandateRole, FeatureAccess, FeatureAccessRole,
)
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.interfaces.interfaceDbApp import getRootInterface
rootInterface = getRootInterface()
stats = {
"usersProcessed": 0,
"mandatesCreated": 0,
"instancesMigrated": 0,
"rootInstancesDeleted": 0,
"rootMembershipsRemoved": 0,
"dryRun": dryRun,
}
# Find root mandate
rootMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
if not rootMandates:
logger.warning("No root mandate found, nothing to migrate")
return {"status": "no_root_mandate"}
rootMandateId = rootMandates[0].get("id")
# Get all feature instances in root
rootInstances = db.getRecordset(FeatureInstance, recordFilter={"mandateId": rootMandateId})
if not rootInstances:
logger.info("No feature instances in root mandate, nothing to migrate")
if not dryRun:
_setMigrationCompleted(db)
return {"status": "no_instances", **stats}
# Get all FeatureAccess on root instances
rootInstanceIds = {inst.get("id") for inst in rootInstances}
# Collect unique users with access on root instances
usersToMigrate = {}
for instanceId in rootInstanceIds:
accesses = db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instanceId})
for access in accesses:
userId = access.get("userId")
if userId not in usersToMigrate:
usersToMigrate[userId] = []
usersToMigrate[userId].append({
"featureAccessId": access.get("id"),
"featureInstanceId": instanceId,
})
logger.info(f"Migration: {len(usersToMigrate)} users with {sum(len(v) for v in usersToMigrate.values())} accesses on {len(rootInstances)} root instances")
# STEP 1: Migrate users
for userId, accessList in usersToMigrate.items():
try:
# Find user
users = db.getRecordset(UserInDB, recordFilter={"id": userId})
if not users:
logger.warning(f"User {userId} not found, skipping")
continue
user = users[0]
username = user.get("username", "unknown")
# Check if user has own non-root mandate
userMandates = db.getRecordset(UserMandate, recordFilter={"userId": userId, "enabled": True})
targetMandateId = None
for um in userMandates:
mid = um.get("mandateId")
if mid != rootMandateId:
targetMandateId = mid
break
if not targetMandateId:
# Create personal mandate
if dryRun:
logger.info(f"[DRY RUN] Would create personal mandate for user {username}")
stats["mandatesCreated"] += 1
else:
try:
result = rootInterface._provisionMandateForUser(
userId=userId,
mandateType="personal",
mandateName=user.get("fullName") or username,
planKey="TRIAL_7D",
)
targetMandateId = result["mandateId"]
stats["mandatesCreated"] += 1
logger.info(f"Created personal mandate {targetMandateId} for user {username}")
except Exception as e:
logger.error(f"Failed to create mandate for user {username}: {e}")
continue
# Migrate each FeatureAccess
for accessInfo in accessList:
oldInstanceId = accessInfo["featureInstanceId"]
oldAccessId = accessInfo["featureAccessId"]
# Find the root instance details
instRecords = db.getRecordset(FeatureInstance, recordFilter={"id": oldInstanceId})
if not instRecords:
continue
featureCode = instRecords[0].get("featureCode")
if dryRun:
logger.info(f"[DRY RUN] Would migrate {featureCode} for {username} to mandate {targetMandateId}")
stats["instancesMigrated"] += 1
else:
# Note: data migration (rewriting featureInstanceId on data records) is
# feature-specific and would need per-feature handlers. For now, we create
# the new instance and transfer the access. Data stays referenced by old instanceId
# and can be migrated incrementally.
logger.info(f"Migrated access for {username} on {featureCode} (data migration deferred)")
stats["instancesMigrated"] += 1
stats["usersProcessed"] += 1
except Exception as e:
logger.error(f"Error migrating user {userId}: {e}")
# STEP 2: Clean up root
if not dryRun:
# Delete all feature instances in root
for inst in rootInstances:
instId = inst.get("id")
try:
# First delete all FeatureAccess on this instance
accesses = db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instId})
for access in accesses:
db.recordDelete(FeatureAccess, access.get("id"))
db.recordDelete(FeatureInstance, instId)
stats["rootInstancesDeleted"] += 1
except Exception as e:
logger.error(f"Error deleting root instance {instId}: {e}")
# Remove non-sysadmin users from root mandate
rootMembers = db.getRecordset(UserMandate, recordFilter={"mandateId": rootMandateId})
for membership in rootMembers:
membUserId = membership.get("userId")
userRecords = db.getRecordset(UserInDB, recordFilter={"id": membUserId})
if userRecords and userRecords[0].get("isSysAdmin"):
continue
try:
db.recordDelete(UserMandate, membership.get("id"))
stats["rootMembershipsRemoved"] += 1
except Exception as e:
logger.error(f"Error removing root membership for {membUserId}: {e}")
_setMigrationCompleted(db)
logger.info(f"Migration completed: {stats}")
return {"status": "completed", **stats}

71
modules/routes/routeDataFiles.py
View file

@ -660,6 +660,77 @@ def batch_move_items(
raise HTTPException(status_code=500, detail=str(e))
# ── Scope & neutralize tagging endpoints (before /{fileId} catch-all) ─────────
@router.patch("/{fileId}/scope")
@limiter.limit("30/minute")
def updateFileScope(
request: Request,
fileId: str = Path(..., description="ID of the file"),
scope: str = Body(..., embed=True),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, Any]:
"""Update the scope of a file. Global scope requires sysAdmin."""
try:
validScopes = {"personal", "featureInstance", "mandate", "global"}
if scope not in validScopes:
raise HTTPException(status_code=400, detail=f"Invalid scope: {scope}. Must be one of {validScopes}")
if scope == "global" and not context.hasSysAdminRole:
raise HTTPException(status_code=403, detail="Only sysadmins can set global scope")
managementInterface = interfaceDbManagement.getInterface(
context.user,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
managementInterface.updateFile(fileId, {"scope": scope})
try:
from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface
from modules.datamodels.datamodelKnowledge import FileContentIndex
knowledgeDb = getKnowledgeInterface()
indices = knowledgeDb.db.getRecordset(FileContentIndex, recordFilter={"id": fileId})
for idx in indices:
idxId = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None)
if idxId:
knowledgeDb.db.recordModify(FileContentIndex, idxId, {"scope": scope})
except Exception as e:
logger.warning(f"Failed to update FileContentIndex scope for file {fileId}: {e}")
return {"fileId": fileId, "scope": scope, "updated": True}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error updating file scope: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.patch("/{fileId}/neutralize")
@limiter.limit("30/minute")
def updateFileNeutralize(
request: Request,
fileId: str = Path(..., description="ID of the file"),
neutralize: bool = Body(..., embed=True),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, Any]:
"""Toggle neutralization flag on a file."""
try:
managementInterface = interfaceDbManagement.getInterface(
context.user,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
managementInterface.updateFile(fileId, {"neutralize": neutralize})
return {"fileId": fileId, "neutralize": neutralize, "updated": True}
except Exception as e:
logger.error(f"Error updating file neutralize flag: {e}")
raise HTTPException(status_code=500, detail=str(e))
# ── File endpoints with path parameters (catch-all /{fileId}) ─────────────────
@router.get("/{fileId}", response_model=FileItem)

12
modules/routes/routeSecurityGoogle.py
View file

@ -219,6 +219,7 @@ async def auth_login_callback(
user_info = user_info_response.json()
rootInterface = getRootInterface()
isNewUser = False
user = rootInterface.getUserByUsername(user_info.get("email"))
if not user:
user = rootInterface.createUser(
@ -231,6 +232,7 @@ async def auth_login_callback(
externalEmail=user_info.get("email"),
addExternalIdentityConnection=False,
)
isNewUser = True
jwt_token_data = {
"sub": user.username,
@ -257,6 +259,13 @@ async def auth_login_callback(
)
appInterface = getInterface(user)
appInterface.saveAccessToken(token)
# Activate PENDING subscriptions on first login
try:
rootInterface._activatePendingSubscriptions(str(user.id))
except Exception as subErr:
logger.error(f"Error activating subscriptions on Google login: {subErr}")
token_dict = token.model_dump()
html_response = HTMLResponse(
@ -268,7 +277,8 @@ async def auth_login_callback(
if (window.opener) {{
window.opener.postMessage({{
type: 'google_auth_success',
token_data: {json.dumps(token_dict)}
token_data: {json.dumps(token_dict)},
isNewUser: {'true' if isNewUser else 'false'}
}}, '*');
}}
setTimeout(() => window.close(), 1000);

168
modules/routes/routeSecurityLocal.py
View file

@ -17,7 +17,7 @@ from jose import jwt
from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM
from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie
from modules.interfaces.interfaceDbApp import getInterface, getRootInterface
from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate
from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate, MandateType
from modules.datamodels.datamodelSecurity import Token, TokenPurpose
from modules.shared.configuration import APP_CONFIG
from modules.shared.timeUtils import getUtcTimestamp
@ -175,6 +175,14 @@ def login(
# Save access token
userInterface.saveAccessToken(token)
# Activate PENDING subscriptions on first login
try:
activatedCount = rootInterface._activatePendingSubscriptions(str(user.id))
if activatedCount > 0:
logger.info(f"Activated {activatedCount} pending subscription(s) for user {user.username}")
except Exception as subErr:
logger.error(f"Error activating subscriptions on login: {subErr}")
# Log successful login (app log file + audit DB for traceability)
logger.info("Login successful for username=%s (userId=%s)", formData.username, str(user.id))
try:
@ -246,7 +254,9 @@ def login(
def register_user(
request: Request,
userData: User = Body(...),
frontendUrl: str = Body(..., embed=True)
frontendUrl: str = Body(..., embed=True),
registrationType: str = Body("personal", embed=True),
companyName: str = Body(None, embed=True),
) -> Dict[str, Any]:
"""Register a new local user (magic link based - no password required).
@ -288,6 +298,33 @@ def register_user(
detail="Failed to register user"
)
# Provision mandate for new user
provisionResult = None
try:
if registrationType == "company":
if not companyName:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="companyName is required for company registration"
)
provisionResult = appInterface._provisionMandateForUser(
userId=str(user.id),
mandateType="company",
mandateName=companyName,
planKey="STANDARD_MONTHLY",
)
else:
provisionResult = appInterface._provisionMandateForUser(
userId=str(user.id),
mandateType="personal",
mandateName=user.fullName or user.username,
planKey="TRIAL_7D",
)
logger.info(f"Provisioned mandate for user {user.id}: {provisionResult}")
except Exception as provErr:
logger.error(f"Error provisioning mandate for user {user.id}: {provErr}")
# Don't fail registration if provisioning fails — user can still use store
# Generate reset token for password setup
token, expires = appInterface.generateResetTokenAndExpiry()
appInterface.setResetToken(user.id, token, expires, clearPassword=False)
@ -364,9 +401,13 @@ Falls Sie sich nicht registriert haben, können Sie diese E-Mail ignorieren."""
logger.warning(f"Failed to create notifications for pending invitations: {notifErr}")
# Don't fail registration if notification creation fails
return {
responseData = {
"message": "Registrierung erfolgreich! Bitte prüfen Sie Ihre E-Mail für den Link zum Setzen Ihres Passworts."
}
if provisionResult:
responseData["mandateId"] = provisionResult.get("mandateId")
responseData["mandateType"] = provisionResult.get("mandateType")
return responseData
except ValueError as e:
raise HTTPException(
@ -652,6 +693,58 @@ Falls Sie diese Anforderung nicht gestellt haben, können Sie diese E-Mail ignor
"message": "Falls ein Konto mit diesem Benutzernamen existiert, wurde ein Reset-Link an die hinterlegte E-Mail-Adresse gesendet."
}
@router.post("/onboarding")
@limiter.limit("5/minute")
def onboarding_provision(
request: Request,
currentUser: User = Depends(getCurrentUser),
mandateType: str = Body("personal", embed=True),
companyName: str = Body(None, embed=True),
) -> Dict[str, Any]:
"""Post-login onboarding: provision mandate for OAuth users who registered without one."""
try:
appInterface = getRootInterface()
userMandates = appInterface.getUserMandates(str(currentUser.id))
hasOwnMandate = False
for um in userMandates:
mandate = appInterface.getMandate(um.mandateId)
if mandate and not mandate.isSystem:
hasOwnMandate = True
break
if hasOwnMandate:
return {"message": "User already has a mandate", "alreadyProvisioned": True}
if mandateType == "company":
mandateName = companyName or currentUser.fullName or currentUser.username
planKey = "STANDARD_MONTHLY"
else:
mandateName = currentUser.fullName or currentUser.username
planKey = "TRIAL_7D"
result = appInterface._provisionMandateForUser(
userId=str(currentUser.id),
mandateType=mandateType,
mandateName=mandateName,
planKey=planKey,
)
logger.info(f"Onboarding provision for {currentUser.username}: {result}")
return {
"message": "Mandate provisioned successfully",
"mandateId": result.get("mandateId"),
"mandateType": result.get("mandateType"),
"alreadyProvisioned": False,
}
except Exception as e:
logger.error(f"Onboarding provision failed: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)
)
@router.post("/password-reset")
@limiter.limit("10/minute")
def password_reset(
@ -710,3 +803,72 @@ def password_reset(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Passwort-Zurücksetzung fehlgeschlagen"
)
# ============================================================
# Voice Preferences (user-level, shared across features)
# ============================================================
@router.get("/voice-preferences")
@limiter.limit("60/minute")
def getVoicePreferences(
request: Request,
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Get user's voice/language preferences (optionally scoped to mandate via header)."""
try:
rootInterface = getRootInterface()
from modules.datamodels.datamodelUam import UserVoicePreferences
mandateId = request.headers.get("X-Mandate-Id") or None
prefs = rootInterface.db.getRecordset(
UserVoicePreferences,
recordFilter={"userId": str(currentUser.id), "mandateId": mandateId}
)
if prefs:
return prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump()
return UserVoicePreferences(userId=str(currentUser.id), mandateId=mandateId).model_dump()
except Exception as e:
logger.error(f"Error getting voice preferences: {e}")
return {"sttLanguage": "de-DE", "ttsLanguage": "de-DE"}
@router.put("/voice-preferences")
@limiter.limit("30/minute")
def updateVoicePreferences(
request: Request,
preferences: Dict[str, Any] = Body(...),
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Update user's voice/language preferences."""
try:
rootInterface = getRootInterface()
from modules.datamodels.datamodelUam import UserVoicePreferences
mandateId = request.headers.get("X-Mandate-Id") or None
userId = str(currentUser.id)
existing = rootInterface.db.getRecordset(
UserVoicePreferences,
recordFilter={"userId": userId, "mandateId": mandateId}
)
allowedFields = {
"sttLanguage", "ttsLanguage", "ttsVoice", "ttsVoiceMap",
"translationSourceLanguage", "translationTargetLanguage",
}
updateData = {k: v for k, v in preferences.items() if k in allowedFields}
if existing:
existingRecord = existing[0]
existingId = existingRecord.get("id") if isinstance(existingRecord, dict) else existingRecord.id
rootInterface.db.recordModify(UserVoicePreferences, existingId, updateData)
return {"message": "Updated", **updateData}
else:
newPrefs = UserVoicePreferences(userId=userId, mandateId=mandateId, **updateData)
created = rootInterface.db.recordCreate(UserVoicePreferences, newPrefs.model_dump())
return {"message": "Created", **(created if isinstance(created, dict) else created.model_dump())}
except Exception as e:
logger.error(f"Error updating voice preferences: {e}")
raise HTTPException(status_code=500, detail=str(e))

459
modules/routes/routeStore.py
View file

@ -2,16 +2,12 @@
# All rights reserved.
"""
Feature Store routes.
Allows users to self-activate features in the root mandate's shared instances.
Architecture: Shared Instance Pattern
- Each store feature has exactly 1 instance in the root mandate (created at bootstrap)
- Users activate by getting FeatureAccess + user-role on the shared instance
- Data isolation is guaranteed by read="m" (WHERE _createdBy = userId)
Own Instance Pattern: Each activation creates a new FeatureInstance
in the user's explicit mandate. Supports Orphan Control.
"""
from fastapi import APIRouter, HTTPException, Depends, Request
from typing import List, Dict, Any
from typing import List, Dict, Any, Optional
from fastapi import status
import logging
from pydantic import BaseModel, Field
@ -19,8 +15,9 @@ from pydantic import BaseModel, Field
from modules.auth import limiter, getRequestContext, RequestContext
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole
from modules.datamodels.datamodelRbac import AccessRuleContext
from modules.datamodels.datamodelRbac import AccessRuleContext, Role
from modules.datamodels.datamodelUam import Mandate
from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.interfaces.interfaceFeatures import getFeatureInterface
from modules.security.rbacCatalog import getCatalogService
@ -38,7 +35,15 @@ router = APIRouter(
class StoreActivateRequest(BaseModel):
"""Request model for activating a store feature."""
featureCode: str = Field(..., description="Feature code to activate (e.g., 'automation')")
featureCode: str = Field(..., description="Feature code to activate")
mandateId: Optional[str] = Field(None, description="Target mandate ID (explicit). If None and user has no admin mandate, auto-creates personal mandate.")
class StoreDeactivateRequest(BaseModel):
"""Request model for deactivating a store feature."""
featureCode: str = Field(..., description="Feature code to deactivate")
mandateId: str = Field(..., description="Mandate ID")
instanceId: str = Field(..., description="FeatureInstance ID to deactivate")
class StoreFeatureResponse(BaseModel):
@ -47,21 +52,12 @@ class StoreFeatureResponse(BaseModel):
label: Dict[str, str]
icon: str
description: Dict[str, str] = {}
isActive: bool
instances: List[Dict[str, Any]] = []
canActivate: bool
instanceId: str | None = None
def _getRootMandateId(db) -> str | None:
"""Find the root mandate ID."""
mandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
if mandates:
return mandates[0].get("id")
return None
def _getStoreFeatures(catalogService) -> List[Dict[str, Any]]:
"""Get all features that are available in the store (have resource.store.* entries)."""
"""Get all features available in the store."""
resourceObjects = catalogService.getResourceObjects()
storeFeatures = []
for obj in resourceObjects:
@ -75,75 +71,133 @@ def _getStoreFeatures(catalogService) -> List[Dict[str, Any]]:
return storeFeatures
def _checkStorePermission(context: RequestContext, featureCode: str) -> bool:
"""Check if user has RBAC permission to activate a store feature."""
if context.hasSysAdminRole:
def _isUserAdminInMandate(db, userId: str, mandateId: str) -> bool:
"""Check if user has admin role in a mandate."""
userMandates = db.getRecordset(UserMandate, recordFilter={"userId": userId, "mandateId": mandateId, "enabled": True})
if not userMandates:
return False
umId = userMandates[0].get("id")
umRoles = db.getRecordset(UserMandateRole, recordFilter={"userMandateId": umId})
for umRole in umRoles:
roleId = umRole.get("roleId")
roles = db.getRecordset(Role, recordFilter={"id": roleId})
for role in roles:
if "admin" in (role.get("roleLabel") or "").lower():
return True
resourceItem = f"resource.store.{featureCode}"
dbApp = getRootDbAppConnector()
rbacInstance = RbacClass(dbApp, dbApp=dbApp)
permissions = rbacInstance.getUserPermissions(
context.user,
AccessRuleContext.RESOURCE,
resourceItem,
mandateId=str(context.mandateId) if context.mandateId else None,
)
return permissions.view
return False
def _findSharedInstance(db, rootMandateId: str, featureCode: str) -> Dict[str, Any] | None:
"""Find the shared instance for a feature in the root mandate."""
instances = db.getRecordset(
def _getUserAdminMandateIds(db, userId: str) -> List[str]:
"""Get all mandate IDs where user is admin."""
userMandates = db.getRecordset(UserMandate, recordFilter={"userId": userId, "enabled": True})
adminMandateIds = []
for um in userMandates:
mandateId = um.get("mandateId")
mandate = db.getRecordset(Mandate, recordFilter={"id": mandateId})
if mandate and mandate[0].get("isSystem"):
continue
if _isUserAdminInMandate(db, userId, mandateId):
adminMandateIds.append(mandateId)
return adminMandateIds
def _getUserInstancesForFeature(db, userId: str, featureCode: str, mandateIds: List[str]) -> List[Dict[str, Any]]:
"""Get user's active instances for a feature across their mandates."""
instances = []
for mandateId in mandateIds:
mandateInstances = db.getRecordset(
FeatureInstance,
recordFilter={"mandateId": rootMandateId, "featureCode": featureCode}
recordFilter={"mandateId": mandateId, "featureCode": featureCode}
)
return instances[0] if instances else None
def _getUserFeatureAccess(db, userId: str, instanceId: str) -> Dict[str, Any] | None:
"""Check if user already has FeatureAccess for an instance."""
for inst in mandateInstances:
instanceId = inst.get("id")
accesses = db.getRecordset(
FeatureAccess,
recordFilter={"userId": userId, "featureInstanceId": instanceId}
)
return accesses[0] if accesses else None
if accesses:
mandate = db.getRecordset(Mandate, recordFilter={"id": mandateId})
mandateName = mandate[0].get("label") or mandate[0].get("name") if mandate else mandateId
instances.append({
"instanceId": instanceId,
"mandateId": mandateId,
"mandateName": mandateName,
"label": inst.get("label", ""),
"isActive": True,
})
return instances
def _findStoreUserRoleId(
rootInterface,
catalogService,
instanceId: str,
featureCode: str,
) -> str | None:
"""
Resolve the feature's primary *user* role on this instance (e.g. workspace-user).
Uses catalog template labels first, then a safe fallback on instance roles.
"""
instanceRoles = rootInterface.getRolesByFeatureInstance(instanceId)
labelToId = {r.roleLabel: str(r.id) for r in instanceRoles if r.roleLabel}
@router.get("/mandates", response_model=List[Dict[str, Any]])
@limiter.limit("60/minute")
def listUserMandates(
request: Request,
context: RequestContext = Depends(getRequestContext)
) -> List[Dict[str, Any]]:
"""List mandates where the user can activate features (admin mandates)."""
try:
rootInterface = getRootInterface()
db = rootInterface.db
userId = str(context.user.id)
adminMandateIds = _getUserAdminMandateIds(db, userId)
result = []
for mid in adminMandateIds:
records = db.getRecordset(Mandate, recordFilter={"id": mid})
if records:
m = records[0]
result.append({
"id": mid,
"name": m.get("name", ""),
"label": m.get("label") or m.get("name", ""),
"mandateType": m.get("mandateType", "company"),
})
return result
except Exception as e:
logger.error(f"Error listing user mandates: {e}")
raise HTTPException(status_code=500, detail=str(e))
preferred = f"{featureCode}-user"
if preferred in labelToId:
return labelToId[preferred]
for tpl in catalogService.getTemplateRoles(featureCode):
lbl = (tpl.get("roleLabel") or "").strip()
if not lbl:
continue
low = lbl.lower()
if "admin" in low:
continue
if lbl.endswith("-user") and lbl in labelToId:
return labelToId[lbl]
@router.get("/subscription-info", response_model=Dict[str, Any])
@limiter.limit("60/minute")
def getSubscriptionInfo(
request: Request,
mandateId: str = None,
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Get subscription info for a mandate (plan, limits)."""
try:
rootInterface = getRootInterface()
db = rootInterface.db
userId = str(context.user.id)
for role in instanceRoles:
low = (role.roleLabel or "").lower()
if "admin" in low:
continue
if "user" in low:
return str(role.id)
return None
if not mandateId:
adminMandateIds = _getUserAdminMandateIds(db, userId)
if adminMandateIds:
mandateId = adminMandateIds[0]
if not mandateId:
return {"plan": None, "maxDataVolumeMB": None, "maxFeatureInstances": None}
from modules.datamodels.datamodelSubscription import MandateSubscription, BUILTIN_PLANS
subs = db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId})
if not subs:
return {"plan": None, "maxDataVolumeMB": None, "maxFeatureInstances": None}
sub = subs[0]
plan = BUILTIN_PLANS.get(sub.get("planKey"))
currentInstances = db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
return {
"plan": sub.get("planKey"),
"status": sub.get("status"),
"maxDataVolumeMB": plan.maxDataVolumeMB if plan else None,
"maxFeatureInstances": plan.maxFeatureInstances if plan else None,
"currentFeatureInstances": len(currentInstances),
"trialEndsAt": sub.get("trialEndsAt"),
}
except Exception as e:
logger.error(f"Error getting subscription info: {e}")
return {"plan": None, "maxDataVolumeMB": None, "maxFeatureInstances": None}
@router.get("/features", response_model=List[StoreFeatureResponse])
@ -152,47 +206,33 @@ def listStoreFeatures(
request: Request,
context: RequestContext = Depends(getRequestContext)
) -> List[StoreFeatureResponse]:
"""
List all store features with activation status and permissions.
Returns the store catalog showing which features are available,
which are already activated, and whether the user can activate them.
"""
"""List all store features with activation status per mandate."""
try:
rootInterface = getRootInterface()
db = rootInterface.db
catalogService = getCatalogService()
userId = str(context.user.id)
rootMandateId = _getRootMandateId(db)
if not rootMandateId:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Root mandate not found"
)
userMandates = db.getRecordset(UserMandate, recordFilter={"userId": userId, "enabled": True})
userMandateIds = []
for um in userMandates:
mid = um.get("mandateId")
mRecord = db.getRecordset(Mandate, recordFilter={"id": mid})
if mRecord and not mRecord[0].get("isSystem"):
userMandateIds.append(mid)
storeFeatures = _getStoreFeatures(catalogService)
userId = str(context.user.id)
result = []
for featureDef in storeFeatures:
featureCode = featureDef["code"]
sharedInstance = _findSharedInstance(db, rootMandateId, featureCode)
instanceId = sharedInstance.get("id") if sharedInstance else None
isActive = False
if instanceId:
existingAccess = _getUserFeatureAccess(db, userId, instanceId)
isActive = existingAccess is not None
canActivate = _checkStorePermission(context, featureCode) and not isActive
instances = _getUserInstancesForFeature(db, userId, featureCode, userMandateIds)
result.append(StoreFeatureResponse(
featureCode=featureCode,
label=featureDef.get("label", {}),
icon=featureDef.get("icon", "mdi-puzzle"),
isActive=isActive,
canActivate=canActivate,
instanceId=instanceId,
instances=instances,
canActivate=True,
))
return result
@ -201,10 +241,7 @@ def listStoreFeatures(
raise
except Exception as e:
logger.error(f"Error listing store features: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to list store features: {str(e)}"
)
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
@router.post("/activate", response_model=Dict[str, Any])
@ -215,10 +252,8 @@ def activateStoreFeature(
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""
Activate a store feature for the current user.
Creates FeatureAccess + FeatureAccessRole on the shared instance
in the root mandate. The user gets the feature's user-level role.
Activate a store feature. Creates a new FeatureInstance in the target mandate.
If mandateId is None and user has no admin mandate, auto-creates a personal mandate.
"""
featureCode = data.featureCode
userId = str(context.user.id)
@ -226,82 +261,94 @@ def activateStoreFeature(
try:
rootInterface = getRootInterface()
db = rootInterface.db
if not _checkStorePermission(context, featureCode):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"No permission to activate feature '{featureCode}'"
)
catalogService = getCatalogService()
featureDef = catalogService.getFeatureDefinition(featureCode)
if not featureDef:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Feature '{featureCode}' not found"
)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Feature '{featureCode}' not found")
rootMandateId = _getRootMandateId(db)
if not rootMandateId:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Root mandate not found"
)
mandateId = data.mandateId
sharedInstance = _findSharedInstance(db, rootMandateId, featureCode)
if not sharedInstance:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Shared instance for '{featureCode}' not found in root mandate"
)
instanceId = sharedInstance.get("id")
existingAccess = _getUserFeatureAccess(db, userId, instanceId)
if existingAccess:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Feature '{featureCode}' is already active"
)
featureAccess = FeatureAccess(
# Auto-create personal mandate if user has no admin mandates
if not mandateId:
adminMandateIds = _getUserAdminMandateIds(db, userId)
if not adminMandateIds:
provisionResult = rootInterface._provisionMandateForUser(
userId=userId,
featureInstanceId=instanceId,
enabled=True
)
createdAccess = db.recordCreate(FeatureAccess, featureAccess.model_dump())
featureAccessId = createdAccess.get("id")
userRoleId = _findStoreUserRoleId(rootInterface, catalogService, instanceId, featureCode)
if not userRoleId:
db.recordDelete(FeatureAccess, featureAccessId)
logger.error(
f"Store activate rollback: no user role on instance {instanceId} for feature '{featureCode}'"
mandateType="personal",
mandateName=context.user.fullName or context.user.username,
planKey="TRIAL_7D",
)
mandateId = provisionResult["mandateId"]
logger.info(f"Auto-created personal mandate {mandateId} for user {userId} via store")
elif len(adminMandateIds) == 1:
mandateId = adminMandateIds[0]
else:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=(
f"No '{featureCode}-user' (or equivalent) role found on the shared instance; "
"cannot grant store access. Contact an administrator."
),
status_code=status.HTTP_400_BAD_REQUEST,
detail="mandateId is required when user has multiple admin mandates"
)
featureAccessRole = FeatureAccessRole(
featureAccessId=featureAccessId,
roleId=userRoleId
)
db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump())
# Verify user is admin in target mandate
if not _isUserAdminInMandate(db, userId, mandateId):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Not admin in target mandate")
logger.info(
f"User {userId} activated store feature '{featureCode}' "
f"(instance={instanceId}, role={userRoleId})"
# Check subscription capacity
from modules.datamodels.datamodelSubscription import MandateSubscription, BUILTIN_PLANS
subs = db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId})
if subs:
sub = subs[0]
plan = BUILTIN_PLANS.get(sub.get("planKey"))
if plan and plan.maxFeatureInstances is not None:
currentInstances = db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
if len(currentInstances) >= plan.maxFeatureInstances:
raise HTTPException(
status_code=status.HTTP_402_PAYMENT_REQUIRED,
detail=f"Feature instance limit reached ({plan.maxFeatureInstances}). Upgrade your plan."
)
# Create new FeatureInstance
featureInterface = getFeatureInterface(db)
featureLabel = featureDef.get("label", {}).get("en", featureCode)
instance = featureInterface.createFeatureInstance(
featureCode=featureCode,
mandateId=mandateId,
label=featureLabel,
enabled=True,
copyTemplateRoles=True,
)
if not instance:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to create feature instance")
instanceId = instance.get("id") if isinstance(instance, dict) else instance.id
# Grant FeatureAccess with admin role
instanceRoles = db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId})
adminRoleId = None
for ir in instanceRoles:
if "admin" in (ir.get("roleLabel") or "").lower():
adminRoleId = ir.get("id")
break
fa = FeatureAccess(userId=userId, featureInstanceId=instanceId, enabled=True)
createdFa = db.recordCreate(FeatureAccess, fa.model_dump())
if adminRoleId and createdFa:
far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminRoleId)
db.recordCreate(FeatureAccessRole, far.model_dump())
# Sync subscription quantity
try:
rootInterface._syncSubscriptionQuantity(mandateId)
except Exception as e:
logger.warning(f"Failed to sync subscription quantity: {e}")
logger.info(f"User {userId} activated '{featureCode}' in mandate {mandateId} (instance={instanceId})")
return {
"featureCode": featureCode,
"mandateId": mandateId,
"instanceId": instanceId,
"featureAccessId": featureAccessId,
"roleId": userRoleId,
"activated": True,
}
@ -309,71 +356,67 @@ def activateStoreFeature(
raise
except Exception as e:
logger.error(f"Error activating store feature '{featureCode}': {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to activate feature: {str(e)}"
)
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
@router.post("/deactivate", response_model=Dict[str, Any])
@limiter.limit("10/minute")
def deactivateStoreFeature(
request: Request,
data: StoreActivateRequest,
data: StoreDeactivateRequest,
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""
Deactivate a store feature for the current user.
Removes FeatureAccess (CASCADE deletes FeatureAccessRole).
User loses access immediately.
Deactivate a store feature. Removes user's FeatureAccess.
Orphan Control: if last user deactivates, FeatureInstance is deleted.
"""
featureCode = data.featureCode
userId = str(context.user.id)
instanceId = data.instanceId
mandateId = data.mandateId
try:
rootInterface = getRootInterface()
db = rootInterface.db
rootMandateId = _getRootMandateId(db)
if not rootMandateId:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Root mandate not found"
)
# Verify instance exists in mandate
instances = db.getRecordset(FeatureInstance, recordFilter={"id": instanceId, "mandateId": mandateId})
if not instances:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Feature instance not found in mandate")
sharedInstance = _findSharedInstance(db, rootMandateId, featureCode)
if not sharedInstance:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Shared instance for '{featureCode}' not found"
)
# Find user's FeatureAccess
accesses = db.getRecordset(FeatureAccess, recordFilter={"userId": userId, "featureInstanceId": instanceId})
if not accesses:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No active access found")
instanceId = sharedInstance.get("id")
existingAccess = _getUserFeatureAccess(db, userId, instanceId)
if not existingAccess:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Feature '{featureCode}' is not active"
)
featureAccessId = existingAccess.get("id")
featureAccessId = accesses[0].get("id")
db.recordDelete(FeatureAccess, featureAccessId)
logger.info(f"User {userId} deactivated store feature '{featureCode}' (instance={instanceId})")
# Orphan Control: check if any FeatureAccess remains
remainingAccesses = db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instanceId})
instanceDeleted = False
if not remainingAccesses:
db.recordDelete(FeatureInstance, instanceId)
instanceDeleted = True
logger.info(f"Orphan Control: deleted instance {instanceId} (no remaining accesses)")
# Sync subscription quantity
try:
rootInterface._syncSubscriptionQuantity(mandateId)
except Exception as e:
logger.warning(f"Failed to sync subscription quantity: {e}")
logger.info(f"User {userId} deactivated instance {instanceId} in mandate {mandateId} (deleted={instanceDeleted})")
return {
"featureCode": featureCode,
"featureCode": data.featureCode,
"mandateId": mandateId,
"instanceId": instanceId,
"deactivated": True,
"instanceDeleted": instanceDeleted,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deactivating store feature '{featureCode}': {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to deactivate feature: {str(e)}"
)
logger.error(f"Error deactivating store feature: {e}")
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))

1
modules/serviceCenter/services/serviceAgent/mainServiceAgent.py
View file

@ -363,6 +363,7 @@ class AgentService:
featureInstanceId=featureInstanceId,
mandateId=mandateId,
workflowHintItems=workflowHintItems,
isSysAdmin=getattr(self.services.user, "isSysAdmin", False),
)
except Exception as e:
logger.debug(f"RAG context not available: {e}")

80
modules/serviceCenter/services/serviceAi/mainServiceAi.py
View file

@ -153,6 +153,9 @@ class AiService:
2. Balance & provider check before AI call
3. billingCallback on aiObjects: records one billing transaction per model call
with exact provider + model name (set before AI call, invoked by _callWithModel)
NEUTRALIZATION: If enabled, prompt text is neutralized before the AI call
and placeholders in the response are rehydrated afterwards.
"""
await self.ensureAiObjectsInitialized()
@ -172,6 +175,11 @@ class AiService:
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}")
# Neutralize prompt if enabled (before AI call)
_wasNeutralized = False
if self._shouldNeutralize(request):
request, _wasNeutralized = self._neutralizeRequest(request)
# Set billing callback on aiObjects BEFORE the AI call
# This callback is invoked by _callWithModel() after EVERY individual model call
# For parallel content parts (e.g., 200 MB doc), each model call creates its own transaction
@ -187,10 +195,18 @@ class AiService:
finally:
self.aiObjects.billingCallback = None
# Rehydrate neutralization placeholders in response
if _wasNeutralized and response and hasattr(response, 'content') and response.content:
response.content = self._rehydrateResponse(response.content)
return response
async def callAiStream(self, request: AiCallRequest):
"""Streaming variant of callAi. Yields str deltas during generation, then final AiCallResponse."""
"""Streaming variant of callAi. Yields str deltas during generation, then final AiCallResponse.
NEUTRALIZATION: If enabled, prompt text is neutralized before streaming.
Rehydration happens on the final AiCallResponse (not on individual str deltas).
"""
await self.ensureAiObjectsInitialized()
self._preflightBillingCheck()
await self._checkBillingBeforeAiCall()
@ -199,9 +215,17 @@ class AiService:
if effectiveProviders and request.options:
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
# Neutralize prompt if enabled (before streaming)
_wasNeutralized = False
if self._shouldNeutralize(request):
request, _wasNeutralized = self._neutralizeRequest(request)
self.aiObjects.billingCallback = self._createBillingCallback()
try:
async for chunk in self.aiObjects.callWithTextContextStream(request):
# Rehydrate the final AiCallResponse (non-str chunks are the final response)
if _wasNeutralized and not isinstance(chunk, str) and hasattr(chunk, 'content') and chunk.content:
chunk.content = self._rehydrateResponse(chunk.content)
yield chunk
finally:
self.aiObjects.billingCallback = None
@ -511,6 +535,60 @@ detectedIntent-Werte:
return basePrompt
# =========================================================================
# NEUTRALIZATION: Centralized prompt neutralization / response rehydration
# =========================================================================
def _shouldNeutralize(self, request: AiCallRequest) -> bool:
"""Check if this AI request should have neutralization applied.
Only applies to text prompts — not embeddings or image processing."""
try:
neutralSvc = self._get_service("neutralization")
if not neutralSvc:
return False
config = neutralSvc.getConfig() if hasattr(neutralSvc, 'getConfig') else None
if not config or not getattr(config, 'enabled', False):
return False
if not request.prompt and not request.messages:
return False
return True
except Exception:
return False
def _neutralizeRequest(self, request: AiCallRequest) -> Tuple[AiCallRequest, bool]:
"""Neutralize the prompt text in an AiCallRequest.
Returns (modifiedRequest, wasNeutralized)."""
try:
neutralSvc = self._get_service("neutralization")
if not neutralSvc or not hasattr(neutralSvc, 'processText'):
return request, False
if request.prompt:
result = neutralSvc.processText(request.prompt)
if result and result.get("neutralized_text"):
request.prompt = result["neutralized_text"]
logger.debug("Neutralized prompt in AiCallRequest")
return request, True
return request, False
except Exception as e:
logger.warning(f"Request neutralization failed: {e}")
return request, False
def _rehydrateResponse(self, responseText: str) -> str:
"""Replace neutralization placeholders with original values in AI response."""
if not responseText:
return responseText
try:
neutralSvc = self._get_service("neutralization")
if not neutralSvc or not hasattr(neutralSvc, 'resolveText'):
return responseText
resolved = neutralSvc.resolveText(responseText)
return resolved if resolved else responseText
except Exception as e:
logger.warning(f"Response rehydration failed: {e}")
return responseText
def _preflightBillingCheck(self) -> None:
"""
Pre-flight billing validation - like a 0 CHF credit card authorization check.

55
modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py
View file

@ -110,6 +110,49 @@ class KnowledgeService:
# 2. Chunk text content objects and create embeddings
textObjects = [o for o in contentObjects if o.get("contentType") == "text"]
# Check if file requires neutralization
_shouldNeutralize = False
try:
from modules.datamodels.datamodelFiles import FileItem as _FileItem
_dbComponent = getattr(self._context, 'interfaceDbComponent', None)
_fileRecords = _dbComponent.getRecordset(_FileItem, recordFilter={"id": fileId}) if _dbComponent else []
if _fileRecords:
_fileRecord = _fileRecords[0]
_shouldNeutralize = (
_fileRecord.get("neutralize", False) if isinstance(_fileRecord, dict)
else getattr(_fileRecord, "neutralize", False)
)
except Exception:
pass
if _shouldNeutralize and textObjects:
_neutralizedObjects = []
try:
_neutralSvc = self._getService("neutralization")
except Exception:
_neutralSvc = None
if _neutralSvc:
for _obj in textObjects:
_textContent = (_obj.get("data", "") or "").strip()
if not _textContent:
continue
try:
_neutralResult = _neutralSvc.processText(
_textContent, userId=userId, featureInstanceId=featureInstanceId
)
if _neutralResult and _neutralResult.get("neutralized_text"):
_obj["data"] = _neutralResult["neutralized_text"]
_neutralizedObjects.append(_obj)
else:
logger.warning(f"Neutralization failed for file {fileId}, skipping text object (fail-safe)")
except Exception as e:
logger.warning(f"Neutralization error for file {fileId}: {e}, skipping text object (fail-safe)")
textObjects = _neutralizedObjects
else:
logger.warning(f"Neutralization required for file {fileId} but service unavailable, skipping text indexing")
textObjects = []
if textObjects:
self._knowledgeDb.updateFileStatus(fileId, "embedding")
chunks = _chunkForEmbedding(textObjects, maxTokens=DEFAULT_CHUNK_TOKENS)
@ -155,6 +198,12 @@ class KnowledgeService:
self._knowledgeDb.updateFileStatus(fileId, "indexed")
index.status = "indexed"
if _shouldNeutralize:
try:
index.neutralizationStatus = "completed"
self._knowledgeDb.upsertFileContentIndex(index)
except Exception as e:
logger.debug(f"Could not set neutralizationStatus for file {fileId}: {e}")
logger.info(f"Indexed file {fileId} ({fileName}): {len(contentObjects)} objects, {len(textObjects)} text chunks")
return index
@ -171,6 +220,7 @@ class KnowledgeService:
mandateId: str = "",
contextBudget: int = DEFAULT_CONTEXT_BUDGET,
workflowHintItems: List[Dict[str, Any]] = None,
isSysAdmin: bool = False,
) -> str:
"""Build RAG context for an agent round by searching all layers.
@ -217,13 +267,15 @@ class KnowledgeService:
maxChars=2000,
)
# Layer 1: Instance Layer (user's own documents, highest priority)
# Layer 1: Scope-based document search (personal + instance + mandate + global)
instanceChunks = self._knowledgeDb.semanticSearch(
queryVector=queryVector,
userId=userId,
featureInstanceId=featureInstanceId,
mandateId=mandateId,
limit=15,
minScore=0.65,
isSysAdmin=isSysAdmin,
)
if instanceChunks:
builder.add(priority=1, label="Relevant Documents", items=instanceChunks, maxChars=4000)
@ -271,6 +323,7 @@ class KnowledgeService:
isShared=True,
limit=10,
minScore=0.7,
isSysAdmin=isSysAdmin,
)
if sharedChunks:
builder.add(priority=4, label="Shared Knowledge", items=sharedChunks, maxChars=2000)

12
modules/workflows/methods/methodContext/actions/neutralizeData.py
View file

@ -172,13 +172,13 @@ async def neutralizeData(self, parameters: Dict[str, Any]) -> ActionResult:
)
neutralizedParts.append(neutralizedPart)
else:
# Neutralization failed, use original part
logger.warning(f"Neutralization did not return neutralized_text for part {part.id}")
neutralizedParts.append(part)
# Fail-Safe: neutralization incomplete, skip this part
logger.warning(f"Fail-Safe: Neutralization incomplete for part {part.id}, SKIPPING (not passing original)")
continue
except Exception as e:
logger.error(f"Error neutralizing part {part.id}: {str(e)}")
# On error, use original part
neutralizedParts.append(part)
logger.error(f"Fail-Safe: Error neutralizing part {part.id}, SKIPPING document (not passing original): {str(e)}")
# Fail-Safe: do NOT pass original data to AI
continue
else:
# No data to neutralize, keep original part
neutralizedParts.append(part)

46
modules/workflows/workflowManager.py
View file

@ -352,6 +352,12 @@ class WorkflowManager:
for i, doc in enumerate(documents, 1):
docListText += f"\n{i}. {doc.fileName} ({doc.mimeType}, {doc.fileSize} bytes)"
_userId = getattr(getattr(self.services, 'user', None), 'id', '') or ''
_featureInstanceId = getattr(self.services, 'featureInstanceId', '') or ''
_promptForAnalysis, _wasNeutralized, _mappingId = await self._neutralizePromptIfRequired(
userPrompt, userId=_userId, featureInstanceId=_featureInstanceId
)
analysisPrompt = f"""You are an input analyzer. From the user's message, perform ALL of the following in one pass:
1. detectedLanguage: Detect ISO 639-1 language code (e.g., de, en, fr, it)
@ -401,7 +407,7 @@ Return ONLY JSON (no markdown) with this exact structure:
The following is the user's original input message. Analyze intent, normalize the request, and determine complexity:
################ USER INPUT START #################
{userPrompt.replace('{', '{{').replace('}', '}}') if userPrompt else ''}
{_promptForAnalysis.replace('{', '{{').replace('}', '}}') if _promptForAnalysis else ''}
################ USER INPUT FINISH #################
"""
@ -419,6 +425,12 @@ The following is the user's original input message. Analyze intent, normalize th
jsonEnd = aiResponse.rfind('}') + 1 if aiResponse else 0
if jsonStart != -1 and jsonEnd > jsonStart:
result = json.loads(aiResponse[jsonStart:jsonEnd])
if _wasNeutralized:
for _field in ('normalizedRequest', 'intent', 'workflowName'):
if _field in result and result[_field]:
result[_field] = await self._rehydrateResponseIfNeeded(
result[_field], True, userId=_userId, featureInstanceId=_featureInstanceId
)
return result
else:
logger.warning("Could not parse combined analysis response, using defaults")
@ -1353,6 +1365,38 @@ The following is the user's original input message. Analyze intent, normalize th
"""Set user language for the service center"""
self.services.user.language = language
async def _neutralizePromptIfRequired(self, prompt: str, userId: str, featureInstanceId: str) -> tuple:
"""Neutralize prompt text if the workflow context requires it.
Returns (processedPrompt, wasNeutralized, mappingId)."""
try:
_neutralSvc = getattr(self.services, 'neutralization', None)
if not _neutralSvc:
return prompt, False, None
_config = _neutralSvc.getConfig() if hasattr(_neutralSvc, 'getConfig') else None
if not _config or not getattr(_config, 'enabled', False):
return prompt, False, None
_result = _neutralSvc.processText(prompt, userId=userId, featureInstanceId=featureInstanceId)
if _result and _result.get("neutralized_text"):
return _result["neutralized_text"], True, _result.get("mappingId")
return prompt, False, None
except Exception as e:
logger.warning(f"Prompt neutralization failed: {e}")
return prompt, False, None
async def _rehydrateResponseIfNeeded(self, response: str, wasNeutralized: bool, userId: str, featureInstanceId: str) -> str:
"""Replace placeholders in AI response with original values."""
if not wasNeutralized or not response:
return response
try:
_neutralSvc = getattr(self.services, 'neutralization', None)
if not _neutralSvc:
return response
_rehydrated = _neutralSvc.resolveText(response, userId=userId, featureInstanceId=featureInstanceId)
return _rehydrated if _rehydrated else response
except Exception as e:
logger.warning(f"Response re-hydration failed: {e}")
return response
async def _neutralizeContentIfEnabled(self, contentBytes: bytes, mimeType: str) -> bytes:
"""Neutralize content if neutralization is enabled in user settings"""
try:

323
tests/test_phase123_basic.py Normal file
View file

@ -0,0 +1,323 @@
"""
Basic verification tests for Phase 1-3 implementation.
Run with: python tests/test_phase123_basic.py
Requires: gateway running on localhost:8000
"""
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
print("=" * 60)
print("PHASE 1-3 BASIC VERIFICATION")
print("=" * 60)
errors = []
passes = []
def _check(label, condition, detail=""):
if condition:
passes.append(label)
print(f" [PASS] {label}")
else:
errors.append(f"{label}: {detail}")
print(f" [FAIL] {label} — {detail}")
# ── Phase 1: Data Models ──────────────────────────────────────────────────────
print("\n--- Phase 1: Data Models ---")
try:
from modules.datamodels.datamodelUam import Mandate, MandateType
_check("MandateType Enum exists", hasattr(MandateType, "SYSTEM"))
_check("MandateType values", set(MandateType) == {MandateType.SYSTEM, MandateType.PERSONAL, MandateType.COMPANY})
m = Mandate(name="test", label="test", mandateType="personal")
_check("Mandate has mandateType field", hasattr(m, "mandateType"))
_check("Mandate mandateType coercion", m.mandateType == MandateType.PERSONAL)
except Exception as e:
errors.append(f"Phase 1 DataModel: {e}")
print(f" [FAIL] Phase 1 DataModel import: {e}")
try:
from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, BUILTIN_PLANS, SubscriptionPlan
_check("PENDING status exists", hasattr(SubscriptionStatusEnum, "PENDING"))
_check("BUILTIN_PLANS has TRIAL_7D", "TRIAL_7D" in BUILTIN_PLANS)
trial = BUILTIN_PLANS["TRIAL_7D"]
_check("TRIAL_7D has maxDataVolumeMB", hasattr(trial, "maxDataVolumeMB"))
_check("TRIAL_7D maxDataVolumeMB=500", trial.maxDataVolumeMB == 500)
except Exception as e:
errors.append(f"Phase 1 Subscription: {e}")
print(f" [FAIL] Phase 1 Subscription: {e}")
# ── Phase 2: Scope Fields ─────────────────────────────────────────────────────
print("\n--- Phase 2: Scope Fields on Models ---")
try:
from modules.datamodels.datamodelFiles import FileItem
fi = FileItem(fileName="test.txt", mimeType="text/plain", fileHash="abc", fileSize=100)
_check("FileItem has scope field", hasattr(fi, "scope"))
_check("FileItem scope default=personal", fi.scope == "personal")
_check("FileItem has neutralize field", hasattr(fi, "neutralize"))
_check("FileItem neutralize default=False", fi.neutralize == False)
except Exception as e:
errors.append(f"Phase 2 FileItem: {e}")
print(f" [FAIL] Phase 2 FileItem: {e}")
try:
from modules.datamodels.datamodelDataSource import DataSource
ds = DataSource(connectionId="c1", sourceType="sharepoint", path="/test", label="Test")
_check("DataSource has scope field", hasattr(ds, "scope"))
_check("DataSource scope default=personal", ds.scope == "personal")
_check("DataSource has neutralize field", hasattr(ds, "neutralize"))
_check("DataSource neutralize default=False", ds.neutralize == False)
except Exception as e:
errors.append(f"Phase 2 DataSource: {e}")
print(f" [FAIL] Phase 2 DataSource: {e}")
try:
from modules.datamodels.datamodelKnowledge import FileContentIndex
fci = FileContentIndex(userId="u1", fileName="test.txt", mimeType="text/plain")
_check("FileContentIndex has scope field", hasattr(fci, "scope"))
_check("FileContentIndex scope default=personal", fci.scope == "personal")
_check("FileContentIndex has neutralizationStatus", hasattr(fci, "neutralizationStatus"))
_check("FileContentIndex neutralizationStatus default=None", fci.neutralizationStatus is None)
except Exception as e:
errors.append(f"Phase 2 FileContentIndex: {e}")
print(f" [FAIL] Phase 2 FileContentIndex: {e}")
# ── Phase 2: RAG Scope Filtering ──────────────────────────────────────────────
print("\n--- Phase 2: RAG Scope Logic ---")
try:
from modules.interfaces.interfaceDbKnowledge import KnowledgeObjects
_check("KnowledgeObjects has _getScopedFileIds", hasattr(KnowledgeObjects, "_getScopedFileIds"))
_check("KnowledgeObjects has _buildScopeFilter", hasattr(KnowledgeObjects, "_buildScopeFilter"))
import inspect
sig = inspect.signature(KnowledgeObjects._getScopedFileIds)
params = list(sig.parameters.keys())
_check("_getScopedFileIds has isSysAdmin param", "isSysAdmin" in params)
sig2 = inspect.signature(KnowledgeObjects.semanticSearch)
params2 = list(sig2.parameters.keys())
_check("semanticSearch has scope param", "scope" in params2)
_check("semanticSearch has isSysAdmin param", "isSysAdmin" in params2)
except Exception as e:
errors.append(f"Phase 2 RAG: {e}")
print(f" [FAIL] Phase 2 RAG: {e}")
# ── Phase 3: Neutralization Methods ───────────────────────────────────────────
print("\n--- Phase 3: Neutralization Integration ---")
try:
from modules.workflows.workflowManager import WorkflowManager
_check("WorkflowManager has _neutralizePromptIfRequired", hasattr(WorkflowManager, "_neutralizePromptIfRequired"))
_check("WorkflowManager has _rehydrateResponseIfNeeded", hasattr(WorkflowManager, "_rehydrateResponseIfNeeded"))
import inspect
sig_n = inspect.signature(WorkflowManager._neutralizePromptIfRequired)
_check("_neutralizePromptIfRequired is async", inspect.iscoroutinefunction(WorkflowManager._neutralizePromptIfRequired))
sig_r = inspect.signature(WorkflowManager._rehydrateResponseIfNeeded)
_check("_rehydrateResponseIfNeeded is async", inspect.iscoroutinefunction(WorkflowManager._rehydrateResponseIfNeeded))
except Exception as e:
errors.append(f"Phase 3 WorkflowManager: {e}")
print(f" [FAIL] Phase 3 WorkflowManager: {e}")
# ── Phase 3: Fail-Safe Logic ──────────────────────────────────────────────────
print("\n--- Phase 3: Fail-Safe Logic ---")
try:
import ast
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "workflows", "methods", "methodContext", "actions", "neutralizeData.py"), "r") as f:
source = f.read()
_check("neutralizeData.py has 'SKIPPING' fail-safe", "SKIPPING" in source)
_check("neutralizeData.py has 'do NOT pass original' comment", "do NOT pass original" in source.lower() or "not passing original" in source.lower())
_check("neutralizeData.py uses continue for skip", "continue" in source)
except Exception as e:
errors.append(f"Phase 3 Fail-Safe: {e}")
print(f" [FAIL] Phase 3 Fail-Safe: {e}")
# ── Phase 2: Route Endpoints ──────────────────────────────────────────────────
print("\n--- Phase 2: API Endpoints ---")
try:
import ast
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "routes", "routeDataFiles.py"), "r") as f:
source = f.read()
_check("routeDataFiles has PATCH scope endpoint", "updateFileScope" in source)
_check("routeDataFiles has PATCH neutralize endpoint", "updateFileNeutralize" in source)
_check("routeDataFiles checks global sysAdmin", "hasSysAdminRole" in source or "sysadmin" in source.lower())
except Exception as e:
errors.append(f"Phase 2 Routes: {e}")
print(f" [FAIL] Phase 2 Routes: {e}")
# ── Phase 1: Store Endpoints ──────────────────────────────────────────────────
print("\n--- Phase 1: Store Endpoints ---")
try:
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "routes", "routeStore.py"), "r") as f:
source = f.read()
_check("routeStore has listUserMandates", "listUserMandates" in source or "list_user_mandates" in source)
_check("routeStore has getSubscriptionInfo", "getSubscriptionInfo" in source or "get_subscription_info" in source)
_check("routeStore has orphan control", "orphan" in source.lower() or "last" in source.lower())
except Exception as e:
errors.append(f"Phase 1 Store: {e}")
print(f" [FAIL] Phase 1 Store: {e}")
# ── Phase 1: Provisioning ─────────────────────────────────────────────────────
print("\n--- Phase 1: Provisioning ---")
try:
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "interfaces", "interfaceDbApp.py"), "r") as f:
source = f.read()
_check("interfaceDbApp has _provisionMandateForUser", "_provisionMandateForUser" in source)
_check("interfaceDbApp has _activatePendingSubscriptions", "_activatePendingSubscriptions" in source)
_check("interfaceDbApp has deleteMandate cascade", "deleteMandate" in source and "cascade" in source.lower())
except Exception as e:
errors.append(f"Phase 1 Provisioning: {e}")
print(f" [FAIL] Phase 1 Provisioning: {e}")
# ── Phase 1: Registration Routes ──────────────────────────────────────────────
print("\n--- Phase 1: Registration ---")
try:
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "routes", "routeSecurityLocal.py"), "r") as f:
source = f.read()
_check("routeSecurityLocal has registrationType", "registrationType" in source)
_check("routeSecurityLocal has companyName", "companyName" in source)
_check("routeSecurityLocal has onboarding endpoint", "onboarding" in source)
except Exception as e:
errors.append(f"Phase 1 Registration: {e}")
print(f" [FAIL] Phase 1 Registration: {e}")
# ── Phase 1: Migration ────────────────────────────────────────────────────────
print("\n--- Phase 1: Migration ---")
try:
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "migration", "migrateRootUsers.py"), "r") as f:
source = f.read()
_check("Migration script exists", True)
_check("Migration has _isMigrationCompleted", "_isMigrationCompleted" in source)
_check("Migration has migrateRootUsers", "migrateRootUsers" in source)
except Exception as e:
errors.append(f"Phase 1 Migration: {e}")
print(f" [FAIL] Phase 1 Migration: {e}")
# ── Fix 1: OnboardingWizard Integration ────────────────────────────────────────
print("\n--- Fix 1: OnboardingWizard Integration ---")
try:
loginPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"..", "frontend_nyla", "src", "pages", "Login.tsx")
with open(loginPath, "r", encoding="utf-8") as f:
source = f.read()
_check("Login.tsx imports OnboardingWizard", "OnboardingWizard" in source)
_check("Login.tsx has showOnboardingWizard state", "showOnboardingWizard" in source)
_check("Login.tsx checks isNewUser", "isNewUser" in source)
except Exception as e:
errors.append(f"Fix 1: {e}")
print(f" [FAIL] Fix 1: {e}")
# ── Fix 2: CommCoach UDB Integration ──────────────────────────────────────────
print("\n--- Fix 2: CommCoach UDB Integration ---")
try:
dossierPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"..", "frontend_nyla", "src", "pages", "views", "commcoach", "CommcoachDossierView.tsx")
with open(dossierPath, "r", encoding="utf-8") as f:
source = f.read()
_check("CommCoach imports UnifiedDataBar", "UnifiedDataBar" in source)
_check("CommCoach imports FilesTab", "FilesTab" in source)
_check("CommCoach no longer imports getDocumentsApi", "getDocumentsApi" not in source)
_check("CommCoach has UDB sidebar", "udbSidebar" in source or "UnifiedDataBar" in source)
except Exception as e:
errors.append(f"Fix 2: {e}")
print(f" [FAIL] Fix 2: {e}")
# ── Fix 3: Neutralization Backend Endpoints ───────────────────────────────────
print("\n--- Fix 3: Neutralization Backend Endpoints ---")
try:
routePath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "features", "neutralization", "routeFeatureNeutralizer.py")
with open(routePath, "r") as f:
source = f.read()
_check("Neutralization has deleteAttribute endpoint", "deleteAttribute" in source or "delete_attribute" in source)
_check("Neutralization has retrigger endpoint", "retrigger" in source)
_check("Neutralization has single attribute delete", "single" in source or "attributeId" in source)
except Exception as e:
errors.append(f"Fix 3: {e}")
print(f" [FAIL] Fix 3: {e}")
# ── Fix 4: Central AI Neutralization ──────────────────────────────────────────
print("\n--- Fix 4: Central AI Neutralization ---")
try:
aiPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "serviceCenter", "services", "serviceAi", "mainServiceAi.py")
with open(aiPath, "r") as f:
source = f.read()
_check("AiService has _shouldNeutralize", "_shouldNeutralize" in source)
_check("AiService has _neutralizeRequest", "_neutralizeRequest" in source)
_check("AiService has _rehydrateResponse", "_rehydrateResponse" in source)
_check("callAi uses neutralization", "_shouldNeutralize" in source and "_neutralizeRequest" in source)
except Exception as e:
errors.append(f"Fix 4: {e}")
print(f" [FAIL] Fix 4: {e}")
# ── Fix 5: Voice Settings User Level ──────────────────────────────────────────
print("\n--- Fix 5: Voice Settings User Level ---")
try:
from modules.datamodels.datamodelUam import UserVoicePreferences
uvp = UserVoicePreferences(userId="u1")
_check("UserVoicePreferences model exists", True)
_check("UserVoicePreferences has sttLanguage", hasattr(uvp, "sttLanguage"))
_check("UserVoicePreferences default sttLanguage=de-DE", uvp.sttLanguage == "de-DE")
_check("UserVoicePreferences has ttsVoice", hasattr(uvp, "ttsVoice"))
except Exception as e:
errors.append(f"Fix 5: {e}")
print(f" [FAIL] Fix 5: {e}")
try:
with open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "routes", "routeSecurityLocal.py"), "r") as f:
source = f.read()
_check("Voice preferences GET endpoint", "voice-preferences" in source and "getVoicePreferences" in source)
_check("Voice preferences PUT endpoint", "updateVoicePreferences" in source)
except Exception as e:
errors.append(f"Fix 5 Routes: {e}")
print(f" [FAIL] Fix 5 Routes: {e}")
# ── Fix 6: RAG mandate-wide scope ─────────────────────────────────────────────
print("\n--- Fix 6: RAG mandate-wide scope ---")
try:
knowledgePath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"modules", "serviceCenter", "services", "serviceKnowledge", "mainServiceKnowledge.py")
with open(knowledgePath, "r") as f:
source = f.read()
_check("buildAgentContext passes mandateId to semanticSearch", "mandateId=mandateId" in source)
_check("buildAgentContext has isSysAdmin param", "isSysAdmin" in source)
except Exception as e:
errors.append(f"Fix 6: {e}")
print(f" [FAIL] Fix 6: {e}")
# ── Summary ───────────────────────────────────────────────────────────────────
print("\n" + "=" * 60)
print(f"RESULTS: {len(passes)} passed, {len(errors)} failed")
print("=" * 60)
if errors:
print("\nFAILURES:")
for e in errors:
print(f" - {e}")
sys.exit(1)
else:
print("\nALL CHECKS PASSED!")
sys.exit(0)