diff --git a/app.py b/app.py index b88e6a15..63e5652a 100644 --- a/app.py +++ b/app.py @@ -383,7 +383,7 @@ async def lifespan(app: FastAPI): if settingsCreated > 0: logger.info(f"Billing startup: Created {settingsCreated} missing mandate billing settings") - # Step 2: Ensure all users have billing accounts (for PREPAY_USER mandates) + # Step 2: Ensure all users have billing audit accounts accountsCreated = billingInterface.ensureAllUserAccountsExist() if accountsCreated > 0: logger.info(f"Billing startup: Created {accountsCreated} missing user accounts") @@ -510,7 +510,7 @@ app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) async def _insufficientBalanceHandler(request: Request, exc: Exception): - """HTTP 402 with structured billing hint (PREPAY_USER vs PREPAY_MANDATE).""" + """HTTP 402 with structured billing hint.""" payload = exc.toClientDict() if hasattr(exc, "toClientDict") else {"error": "INSUFFICIENT_BALANCE", "message": str(exc)} return JSONResponse(status_code=402, content={"detail": payload}) @@ -555,6 +555,9 @@ app.include_router(userRouter) from modules.routes.routeDataFiles import router as fileRouter app.include_router(fileRouter) +from modules.routes.routeDataSources import router as dataSourceRouter +app.include_router(dataSourceRouter) + from modules.routes.routeDataPrompts import router as promptRouter app.include_router(promptRouter) @@ -579,6 +582,9 @@ app.include_router(clickupApiRouter) from modules.routes.routeVoiceGoogle import router as voiceGoogleRouter app.include_router(voiceGoogleRouter) +from modules.routes.routeVoiceUser import router as voiceUserRouter +app.include_router(voiceUserRouter) + from modules.routes.routeSecurityAdmin import router as adminSecurityRouter app.include_router(adminSecurityRouter) diff --git a/modules/aicore/aicoreBase.py b/modules/aicore/aicoreBase.py index 70dd67c4..e107beb3 100644 --- a/modules/aicore/aicoreBase.py +++ b/modules/aicore/aicoreBase.py @@ -18,7 +18,9 @@ from typing import List, Dict, Any, Optional, AsyncGenerator, Union from modules.datamodels.datamodelAi import AiModel, AiModelCall, AiModelResponse -_RETRY_AFTER_PATTERN = _re.compile(r"try again in (\d+(?:\.\d+)?)\s*s", _re.IGNORECASE) +_RETRY_AFTER_PATTERN = _re.compile( + r"(?:try again in|retry after)\s+(\d+(?:\.\d+)?)\s*s", _re.IGNORECASE +) def _parseRetryAfterSeconds(message: str) -> float: diff --git a/modules/aicore/aicorePluginPrivateLlm.py b/modules/aicore/aicorePluginPrivateLlm.py index 718c5905..79853652 100644 --- a/modules/aicore/aicorePluginPrivateLlm.py +++ b/modules/aicore/aicorePluginPrivateLlm.py @@ -7,9 +7,9 @@ Connects to the private-llm service running on-premise with Ollama backend. Provides OCR and Vision capabilities via local AI models. Models: -- poweron-ocr-general: Text extraction and OCR (deepseek backend) -- poweron-vision-general: General vision tasks (qwen2.5vl backend) -- poweron-vision-deep: Deep vision analysis (granite3.2 backend) +- poweron-text-general: Text (qwen2.5); NEUTRALIZATION_TEXT + data/plan ops +- poweron-vision-general: Vision (qwen2.5vl); IMAGE_ANALYSE + NEUTRALIZATION_IMAGE +- poweron-vision-deep: Vision (granite3.2); IMAGE_ANALYSE + NEUTRALIZATION_IMAGE Pricing (CHF per call): - Text models: CHF 0.010 @@ -22,7 +22,7 @@ import time from typing import List, Optional, Dict, Any from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG -from .aicoreBase import BaseConnectorAi +from .aicoreBase import BaseConnectorAi, RateLimitExceededException from modules.datamodels.datamodelAi import ( AiModel, PriorityEnum, @@ -245,6 +245,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.DATA_ANALYSE, 8), (OperationTypeEnum.DATA_GENERATE, 8), (OperationTypeEnum.DATA_EXTRACT, 8), + (OperationTypeEnum.NEUTRALIZATION_TEXT, 9), ), version="qwen2.5:7b", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_TEXT_PER_CALL @@ -270,6 +271,7 @@ class AiPrivateLlm(BaseConnectorAi): processingMode=ProcessingModeEnum.ADVANCED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.IMAGE_ANALYSE, 9), + (OperationTypeEnum.NEUTRALIZATION_IMAGE, 9), ), version="qwen2.5vl:7b", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_VISION_PER_CALL @@ -295,6 +297,7 @@ class AiPrivateLlm(BaseConnectorAi): processingMode=ProcessingModeEnum.DETAILED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.IMAGE_ANALYSE, 9), + (OperationTypeEnum.NEUTRALIZATION_IMAGE, 9), ), version="granite3.2-vision", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_VISION_PER_CALL @@ -367,6 +370,9 @@ class AiPrivateLlm(BaseConnectorAi): if response.status_code != 200: errorMessage = f"Private-LLM API error: {response.status_code} - {response.text}" + if response.status_code == 429: + logger.warning(errorMessage) + raise RateLimitExceededException(errorMessage) logger.error(errorMessage) raise HTTPException(status_code=500, detail=errorMessage) @@ -458,6 +464,9 @@ class AiPrivateLlm(BaseConnectorAi): if response.status_code != 200: errorMessage = f"Private-LLM API error: {response.status_code} - {response.text}" + if response.status_code == 429: + logger.warning(errorMessage) + raise RateLimitExceededException(errorMessage) logger.error(errorMessage) raise HTTPException(status_code=500, detail=errorMessage) diff --git a/modules/auth/tokenManager.py b/modules/auth/tokenManager.py index 5740a2ac..940de055 100644 --- a/modules/auth/tokenManager.py +++ b/modules/auth/tokenManager.py @@ -181,7 +181,7 @@ class TokenManager: # Only allow a new refresh if at least 10 minutes passed since the token was created/refreshed try: nowTs = getUtcTimestamp() - createdTs = parseTimestamp(oldToken.createdAt, default=0.0) + createdTs = parseTimestamp(oldToken.sysCreatedAt, default=0.0) secondsSinceLastRefresh = nowTs - createdTs if secondsSinceLastRefresh < 10 * 60: logger.info( diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index f0e38fdd..6bd661b4 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -5,13 +5,14 @@ import re import psycopg2 import psycopg2.extras import logging -from typing import List, Dict, Any, Optional, Union, get_origin, get_args, Type +from typing import List, Dict, Any, Optional, Union, get_origin, get_args, Type, Set, Tuple import uuid from pydantic import BaseModel, Field import threading from modules.shared.timeUtils import getUtcTimestamp from modules.shared.configuration import APP_CONFIG +from modules.datamodels.datamodelBase import PowerOnModel from modules.datamodels.datamodelUam import User, AccessLevel, UserPermissions from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext @@ -20,7 +21,7 @@ logger = logging.getLogger(__name__) # No mapping needed - table name = Pydantic model name exactly -class SystemTable(BaseModel): +class SystemTable(PowerOnModel): """Data model for system table entries""" table_name: str = Field( @@ -157,6 +158,88 @@ def _parseRecordFields(record: Dict[str, Any], fields: Dict[str, str], context: logger.warning(f"Could not parse JSONB field {fieldName}, keeping as string ({context})") +# Legacy column names (historical _* internal names and old camelCase audit fields) -> PowerOn sys* columns. +# Order matters: more specific / underscore names first; first successful copy wins per cell via IS NULL on target. +_LEGACY_FIELD_TO_SYS: Tuple[Tuple[str, str], ...] = ( + ("_createdAt", "sysCreatedAt"), + ("_createdBy", "sysCreatedBy"), + ("_modifiedAt", "sysModifiedAt"), + ("_modifiedBy", "sysModifiedBy"), + ("createdAt", "sysCreatedAt"), + ("creationDate", "sysCreatedAt"), + ("updatedAt", "sysModifiedAt"), + ("lastModified", "sysModifiedAt"), +) + + +def _quotePgIdent(name: str) -> str: + return '"' + str(name).replace('"', '""') + '"' + + +def _resolveColumnCaseInsensitive(cols: Set[str], logicalName: str) -> Optional[str]: + """Match information_schema column_name to logical CamelCase (PG folds unquoted legacy names to lowercase).""" + if not logicalName or not cols: + return None + for c in cols: + if c.lower() == logicalName.lower(): + return c + return None + + +def _pgColumnDataType(cursor, tablePg: str, colPg: str) -> Optional[str]: + cursor.execute( + """ + SELECT data_type FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s AND column_name = %s + """, + (tablePg, colPg), + ) + row = cursor.fetchone() + return row["data_type"] if row else None + + +def _legacySourceToSysSqlExpr(srcIdent: str, srcType: Optional[str], tgtType: Optional[str]) -> str: + """Build RHS for UPDATE sys* = expr from legacy _* column (handles text/timestamp -> double precision).""" + s = _quotePgIdent(srcIdent) + sl = (srcType or "").lower() + tl = (tgtType or "").lower() + if "double" in tl or tl == "real" or tl == "numeric": + if any(x in sl for x in ("double precision", "real", "numeric", "integer", "bigint", "smallint")): + return f"{s}::double precision" + if "timestamp" in sl or sl == "date": + return f"EXTRACT(EPOCH FROM {s}::timestamptz)" + if "text" in sl or "character" in sl or sl == "uuid": + return ( + f"CASE WHEN trim({s}::text) ~ '^[+-]?[0-9]+(\\.[0-9]*)?([eE][+-]?[0-9]+)?$' " + f"THEN trim({s}::text)::double precision " + f"ELSE EXTRACT(EPOCH FROM trim({s}::text)::timestamptz) END" + ) + return s + return s + + +def _listPublicBaseTableNames(cursor) -> List[str]: + cursor.execute( + """ + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + ORDER BY table_name + """ + ) + return [row["table_name"] for row in cursor.fetchall()] + + +def _listTableColumnNames(cursor, tableName: str) -> Set[str]: + cursor.execute( + """ + SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (tableName,), + ) + return {row["column_name"] for row in cursor.fetchall()} + + # Cache connectors by (host, database, port) to avoid duplicate inits for same database. # Thread safety: _connector_cache_lock protects cache access. userId is request-scoped via # contextvars to avoid races when concurrent requests share the same connector. @@ -178,7 +261,7 @@ def _get_cached_connector( userId: str = None, ) -> "DatabaseConnector": """Return cached DatabaseConnector for same (host, database, port) to avoid duplicate PostgreSQL inits. - Uses contextvars for userId so concurrent requests sharing the same connector get correct _createdBy/_modifiedBy. + Uses contextvars for userId so concurrent requests sharing the same connector get correct sysCreatedBy/sysModifiedBy. """ port = int(dbPort) if dbPort is not None else 5432 key = (dbHost, dbDatabase, port) @@ -327,8 +410,10 @@ class DatabaseConnector: id SERIAL PRIMARY KEY, table_name VARCHAR(255) UNIQUE NOT NULL, initial_id VARCHAR(255) NOT NULL, - _createdAt DOUBLE PRECISION, - _modifiedAt DOUBLE PRECISION + "sysCreatedAt" DOUBLE PRECISION, + "sysCreatedBy" VARCHAR(255), + "sysModifiedAt" DOUBLE PRECISION, + "sysModifiedBy" VARCHAR(255) ) """) conn.close() @@ -371,6 +456,63 @@ class DatabaseConnector: logger.warning(f"Connection lost, reconnecting: {e}") self._connect() + def migrateLegacyUnderscoreSysColumns(self) -> int: + """ + Scan all public base tables on this connection's database. Where both a legacy + source column (any case: _createdAt, createdAt, creationDate, …) and the matching + sys* column exist, UPDATE sys* from legacy where sys* IS NULL AND legacy IS NOT NULL. + Idempotent; run after schema adds sys* columns (see _ensureTableExists). + """ + self._ensure_connection() + total = 0 + try: + with self.connection.cursor() as cursor: + tableNames = _listPublicBaseTableNames(cursor) + for table in tableNames: + with self.connection.cursor() as cursor: + cols = _listTableColumnNames(cursor, table) + for legacyLogical, sysLogical in _LEGACY_FIELD_TO_SYS: + src = _resolveColumnCaseInsensitive(cols, legacyLogical) + tgt = _resolveColumnCaseInsensitive(cols, sysLogical) + if not src or not tgt or src == tgt: + continue + try: + with self.connection.cursor() as cursor: + srcType = _pgColumnDataType(cursor, table, src) + tgtType = _pgColumnDataType(cursor, table, tgt) + expr = _legacySourceToSysSqlExpr(src, srcType, tgtType) + tq = _quotePgIdent(table) + tr = _quotePgIdent(tgt) + sr = _quotePgIdent(src) + sql = ( + f"UPDATE {tq} SET {tr} = {expr} " + f"WHERE {tr} IS NULL AND {sr} IS NOT NULL" + ) + cursor.execute(sql) + n = cursor.rowcount + self.connection.commit() + total += n + except Exception as e: + try: + self.connection.rollback() + except Exception: + pass + logger.debug( + f"migrateLegacyUnderscoreSysColumns skip {self.dbDatabase}.{table} " + f"{src}->{tgt}: {e}" + ) + except Exception as e: + logger.error(f"migrateLegacyUnderscoreSysColumns failed on {self.dbDatabase}: {e}") + try: + self.connection.rollback() + except Exception: + pass + if total: + logger.info( + f"migrateLegacyUnderscoreSysColumns: {total} cell(s) in {self.dbDatabase}" + ) + return total + def _initializeSystemTable(self): """Initializes the system table if it doesn't exist yet.""" try: @@ -416,7 +558,7 @@ class DatabaseConnector: for table_name, initial_id in data.items(): cursor.execute( """ - INSERT INTO "_system" ("table_name", "initial_id", "_modifiedAt") + INSERT INTO "_system" ("table_name", "initial_id", "sysModifiedAt") VALUES (%s, %s, %s) """, (table_name, initial_id, getUtcTimestamp()), @@ -448,8 +590,10 @@ class DatabaseConnector: CREATE TABLE "{self._systemTableName}" ( "table_name" VARCHAR(255) PRIMARY KEY, "initial_id" VARCHAR(255), - "_createdAt" DOUBLE PRECISION, - "_modifiedAt" DOUBLE PRECISION + "sysCreatedAt" DOUBLE PRECISION, + "sysCreatedBy" VARCHAR(255), + "sysModifiedAt" DOUBLE PRECISION, + "sysModifiedBy" VARCHAR(255) ) """) logger.info("System table created successfully") @@ -464,10 +608,16 @@ class DatabaseConnector: ) existing_columns = [row["column_name"] for row in cursor.fetchall()] - if "_modifiedAt" not in existing_columns: - cursor.execute( - f'ALTER TABLE "{self._systemTableName}" ADD COLUMN "_modifiedAt" DOUBLE PRECISION' - ) + for sys_col, sys_sql in [ + ("sysCreatedAt", "DOUBLE PRECISION"), + ("sysCreatedBy", "VARCHAR(255)"), + ("sysModifiedAt", "DOUBLE PRECISION"), + ("sysModifiedBy", "VARCHAR(255)"), + ]: + if sys_col not in existing_columns: + cursor.execute( + f'ALTER TABLE "{self._systemTableName}" ADD COLUMN "{sys_col}" {sys_sql}' + ) return True except Exception as e: @@ -484,6 +634,7 @@ class DatabaseConnector: try: self._ensure_connection() + schemaTouched = False with self.connection.cursor() as cursor: # Check if table exists by querying information_schema with case-insensitive search @@ -502,6 +653,7 @@ class DatabaseConnector: logger.info( f"Created table '{table}' with columns from Pydantic model" ) + schemaTouched = True else: # Table exists: ensure all columns from model are present (simple additive migration) try: @@ -518,11 +670,7 @@ class DatabaseConnector: # Desired columns based on model model_fields = _get_model_fields(model_class) - desired_columns = ( - set(["id"]) - | set(model_fields.keys()) - | {"_createdAt", "_modifiedAt", "_createdBy", "_modifiedBy"} - ) + desired_columns = set(["id"]) | set(model_fields.keys()) # Add missing columns for col in sorted(desired_columns - existing_columns): @@ -530,12 +678,6 @@ class DatabaseConnector: if col in ["id"]: continue # primary key exists already sql_type = model_fields.get(col) - if col in ["_createdAt"]: - sql_type = "DOUBLE PRECISION" - elif col in ["_modifiedAt"]: - sql_type = "DOUBLE PRECISION" - elif col in ["_createdBy", "_modifiedBy"]: - sql_type = "VARCHAR(255)" if not sql_type: sql_type = "TEXT" try: @@ -545,6 +687,7 @@ class DatabaseConnector: logger.info( f"Added missing column '{col}' ({sql_type}) to '{table}'" ) + schemaTouched = True except Exception as add_err: logger.warning( f"Could not add column '{col}' to '{table}': {add_err}" @@ -555,6 +698,23 @@ class DatabaseConnector: ) self.connection.commit() + if schemaTouched: + try: + n = self.migrateLegacyUnderscoreSysColumns() + if n: + logger.info( + "After schema change on %s.%s: legacy -> sys* migration wrote %s cell(s)", + self.dbDatabase, + table, + n, + ) + except Exception as mig_err: + logger.error( + "migrateLegacyUnderscoreSysColumns failed after schema change %s.%s: %s", + self.dbDatabase, + table, + mig_err, + ) return True except Exception as e: logger.error(f"Error ensuring table {table} exists: {e}") @@ -594,16 +754,6 @@ class DatabaseConnector: if field_name != "id": # Skip id, already defined columns.append(f'"{field_name}" {sql_type}') - # Add metadata columns - columns.extend( - [ - '"_createdAt" DOUBLE PRECISION', - '"_modifiedAt" DOUBLE PRECISION', - '"_createdBy" VARCHAR(255)', - '"_modifiedBy" VARCHAR(255)', - ] - ) - # Create table sql = f'CREATE TABLE IF NOT EXISTS "{table}" ({", ".join(columns)})' cursor.execute(sql) @@ -626,11 +776,7 @@ class DatabaseConnector: """Save record to normalized table with explicit columns.""" # Get columns from Pydantic model instead of database schema fields = _get_model_fields(model_class) - columns = ( - ["id"] - + [field for field in fields.keys() if field != "id"] - + ["_createdAt", "_createdBy", "_modifiedAt", "_modifiedBy"] - ) + columns = ["id"] + [field for field in fields.keys() if field != "id"] if not columns: logger.error(f"No columns found for table {table}") @@ -648,7 +794,7 @@ class DatabaseConnector: value = filtered_record.get(col) # Handle timestamp fields - store as Unix timestamps (floats) for consistency - if col in ["_createdAt", "_modifiedAt"] and value is not None: + if col in ["sysCreatedAt", "sysModifiedAt"] and value is not None: if isinstance(value, str): # Try to parse string as timestamp try: @@ -690,7 +836,7 @@ class DatabaseConnector: [ f'"{col}" = EXCLUDED."{col}"' for col in columns[1:] - if col not in ["_createdAt", "_createdBy"] + if col not in ["sysCreatedAt", "sysCreatedBy"] ] ) @@ -723,6 +869,10 @@ class DatabaseConnector: logger.error(f"Error loading record {recordId} from table {table}: {e}") return None + def getRecord(self, model_class: type, recordId: str) -> Optional[Dict[str, Any]]: + """Load one row by primary key (routes / services; wraps _loadRecord).""" + return self._loadRecord(model_class, str(recordId)) + def _saveRecord( self, model_class: type, recordId: str, record: Dict[str, Any] ) -> bool: @@ -742,17 +892,19 @@ class DatabaseConnector: if effective_user_id is None: effective_user_id = self.userId currentTime = getUtcTimestamp() - # Set _createdAt and _createdBy if this is a new record (record doesn't have _createdAt) - if "_createdAt" not in record: - record["_createdAt"] = currentTime + # Set sysCreatedAt/sysCreatedBy on first persist; always refresh modified fields. + # Treat None and 0 as unset (legacy rows / bad defaults); model_dump often has sysCreatedAt=None. + createdTs = record.get("sysCreatedAt") + if createdTs is None or createdTs == 0 or createdTs == 0.0: + record["sysCreatedAt"] = currentTime if effective_user_id: - record["_createdBy"] = effective_user_id - elif "_createdBy" not in record or not record.get("_createdBy"): + record["sysCreatedBy"] = effective_user_id + elif not record.get("sysCreatedBy"): if effective_user_id: - record["_createdBy"] = effective_user_id - record["_modifiedAt"] = currentTime + record["sysCreatedBy"] = effective_user_id + record["sysModifiedAt"] = currentTime if effective_user_id: - record["_modifiedBy"] = effective_user_id + record["sysModifiedBy"] = effective_user_id with self.connection.cursor() as cursor: self._save_record(cursor, table, recordId, record, model_class) @@ -840,6 +992,26 @@ class DatabaseConnector: logger.error(f"Error removing initial ID for table {table}: {e}") return False + def buildRbacWhereClause( + self, + permissions: UserPermissions, + currentUser: User, + table: str, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Delegate to interfaceRbac.buildRbacWhereClause (tests and call sites use connector as entry).""" + from modules.interfaces.interfaceRbac import buildRbacWhereClause as _buildRbacWhereClause + + return _buildRbacWhereClause( + permissions, + currentUser, + table, + self, + mandateId=mandateId, + featureInstanceId=featureInstanceId, + ) + def updateContext(self, userId: str) -> None: """Updates the context of the database connector. Sets both instance userId and contextvar for request-scoped use when connector is shared. @@ -992,10 +1164,6 @@ class DatabaseConnector: Returns (where_clause, order_clause, limit_clause, values, count_values). """ fields = _get_model_fields(model_class) - fields["_createdAt"] = "DOUBLE PRECISION" - fields["_modifiedAt"] = "DOUBLE PRECISION" - fields["_createdBy"] = "TEXT" - fields["_modifiedBy"] = "TEXT" validColumns = set(fields.keys()) where_parts: List[str] = [] values: List[Any] = [] @@ -1193,10 +1361,6 @@ class DatabaseConnector: """ table = model_class.__name__ fields = _get_model_fields(model_class) - fields["_createdAt"] = "DOUBLE PRECISION" - fields["_modifiedAt"] = "DOUBLE PRECISION" - fields["_createdBy"] = "TEXT" - fields["_modifiedBy"] = "TEXT" if column not in fields: return [] diff --git a/modules/connectors/connectorVoiceGoogle.py b/modules/connectors/connectorVoiceGoogle.py index ddb0d864..0dbb46a5 100644 --- a/modules/connectors/connectorVoiceGoogle.py +++ b/modules/connectors/connectorVoiceGoogle.py @@ -18,6 +18,11 @@ from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) +# Gemini-TTS speaker IDs from voices.list use short names (e.g. "Kore") and require model_name + prompt. +_GEMINI_TTS_DEFAULT_MODEL = "gemini-2.5-flash-tts" +_GEMINI_TTS_NEUTRAL_PROMPT = "Say the following" + + class ConnectorGoogleSpeech: """ Google Cloud Speech-to-Text and Translation connector. @@ -902,6 +907,13 @@ class ConnectorGoogleSpeech: "error": f"Validation error: {e}" } + def _isGeminiTtsSpeakerVoiceName(self, voiceName: str) -> bool: + """True when voice name is a Gemini-TTS speaker id (no BCP-47 prefix like en-US-...).""" + if not voiceName or not isinstance(voiceName, str): + return False + stripped = voiceName.strip() + return bool(stripped) and "-" not in stripped + async def textToSpeech(self, text: str, languageCode: str = "de-DE", voiceName: str = None) -> Dict[str, Any]: """ Convert text to speech using Google Cloud Text-to-Speech. @@ -917,9 +929,6 @@ class ConnectorGoogleSpeech: try: logger.info(f"Converting text to speech: '{text[:50]}...' in {languageCode}") - # Set up the synthesis input - synthesisInput = texttospeech.SynthesisInput(text=text) - # Build the voice request selectedVoice = voiceName or self._getDefaultVoice(languageCode) @@ -931,11 +940,24 @@ class ConnectorGoogleSpeech: logger.info(f"Using TTS voice: {selectedVoice} for language: {languageCode}") - voice = texttospeech.VoiceSelectionParams( - language_code=languageCode, - name=selectedVoice, - ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL - ) + if self._isGeminiTtsSpeakerVoiceName(selectedVoice): + synthesisInput = texttospeech.SynthesisInput( + text=text, + prompt=_GEMINI_TTS_NEUTRAL_PROMPT, + ) + voice = texttospeech.VoiceSelectionParams( + language_code=languageCode, + name=selectedVoice, + model_name=_GEMINI_TTS_DEFAULT_MODEL, + ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL, + ) + else: + synthesisInput = texttospeech.SynthesisInput(text=text) + voice = texttospeech.VoiceSelectionParams( + language_code=languageCode, + name=selectedVoice, + ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL, + ) # Select the type of audio file to return audioConfig = texttospeech.AudioConfig( @@ -1059,7 +1081,8 @@ class ConnectorGoogleSpeech: "language_codes": list(voice.language_codes) if voice.language_codes else [], "gender": gender, "ssml_gender": voice.ssml_gender.name if voice.ssml_gender else "NEUTRAL", - "natural_sample_rate_hertz": voice.natural_sample_rate_hertz + "natural_sample_rate_hertz": voice.natural_sample_rate_hertz, + "geminiTts": self._isGeminiTtsSpeakerVoiceName(voice.name or ""), } # Include any additional fields if available from Google API diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py index 296500aa..96e05185 100644 --- a/modules/datamodels/datamodelAi.py +++ b/modules/datamodels/datamodelAi.py @@ -22,6 +22,10 @@ class OperationTypeEnum(str, Enum): IMAGE_ANALYSE = "imageAnalyse" IMAGE_GENERATE = "imageGenerate" + # Neutralization (dedicated model selection; text vs vision backends) + NEUTRALIZATION_TEXT = "neutralizationText" + NEUTRALIZATION_IMAGE = "neutralizationImage" + # Web Operations WEB_SEARCH_DATA = "webSearch" # Returns list of URLs only WEB_CRAWL = "webCrawl" # Web crawl for a given URL @@ -168,6 +172,7 @@ class AiCallRequest(BaseModel): contentParts: Optional[List['ContentPart']] = None # Content parts for model-aware chunking messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="OpenAI-style messages for multi-turn agent conversations") tools: Optional[List[Dict[str, Any]]] = Field(default=None, description="Tool definitions for native function calling") + requireNeutralization: Optional[bool] = Field(default=None, description="Per-request neutralization override: True=force, False=skip, None=use config") class AiCallResponse(BaseModel): diff --git a/modules/datamodels/datamodelBase.py b/modules/datamodels/datamodelBase.py new file mode 100644 index 00000000..862f177b --- /dev/null +++ b/modules/datamodels/datamodelBase.py @@ -0,0 +1,68 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Base Pydantic model with system-managed fields (DB + API + UI metadata).""" + +from typing import Optional + +from pydantic import BaseModel, Field + +from modules.shared.attributeUtils import registerModelLabels + + +class PowerOnModel(BaseModel): + sysCreatedAt: Optional[float] = Field( + default=None, + description="Record creation timestamp (UTC, set by system)", + json_schema_extra={ + "frontend_type": "timestamp", + "frontend_readonly": True, + "frontend_required": False, + "frontend_visible": False, + "system": True, + }, + ) + sysCreatedBy: Optional[str] = Field( + default=None, + description="User ID who created this record (set by system)", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False, + "frontend_visible": False, + "system": True, + }, + ) + sysModifiedAt: Optional[float] = Field( + default=None, + description="Record last modification timestamp (UTC, set by system)", + json_schema_extra={ + "frontend_type": "timestamp", + "frontend_readonly": True, + "frontend_required": False, + "frontend_visible": False, + "system": True, + }, + ) + sysModifiedBy: Optional[str] = Field( + default=None, + description="User ID who last modified this record (set by system)", + json_schema_extra={ + "frontend_type": "text", + "frontend_readonly": True, + "frontend_required": False, + "frontend_visible": False, + "system": True, + }, + ) + + +registerModelLabels( + "PowerOnModel", + {"en": "Base Record", "de": "Basisdatensatz"}, + { + "sysCreatedAt": {"en": "Created At", "de": "Erstellt am", "fr": "Cree le"}, + "sysCreatedBy": {"en": "Created By", "de": "Erstellt von", "fr": "Cree par"}, + "sysModifiedAt": {"en": "Modified At", "de": "Geaendert am", "fr": "Modifie le"}, + "sysModifiedBy": {"en": "Modified By", "de": "Geaendert von", "fr": "Modifie par"}, + }, +) diff --git a/modules/datamodels/datamodelBilling.py b/modules/datamodels/datamodelBilling.py index 995ac75d..ccf1f4a1 100644 --- a/modules/datamodels/datamodelBilling.py +++ b/modules/datamodels/datamodelBilling.py @@ -6,24 +6,12 @@ from typing import List, Dict, Any, Optional from enum import Enum from datetime import date, datetime, timezone from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels import uuid - -class BillingModelEnum(str, Enum): - """Billing model types (prepaid only; legacy UNLIMITED in DB maps to PREPAY_MANDATE).""" - PREPAY_MANDATE = "PREPAY_MANDATE" # Prepaid budget shared by all users in mandate - PREPAY_USER = "PREPAY_USER" # Prepaid budget per user within mandate - - -# Nur fuer initRootMandateBilling (Root-Mandant PREPAY_USER + Startguthaben in Settings). -DEFAULT_USER_CREDIT_CHF = 5.0 - - -class AccountTypeEnum(str, Enum): - """Account type for billing accounts.""" - MANDATE = "MANDATE" # Account for entire mandate - USER = "USER" # Account for specific user within mandate +# End-customer price for storage above plan-included volume (CHF per GB per month). +STORAGE_PRICE_PER_GB_CHF = 0.50 class TransactionTypeEnum(str, Enum): @@ -39,6 +27,8 @@ class ReferenceTypeEnum(str, Enum): PAYMENT = "PAYMENT" # Payment/top-up ADMIN = "ADMIN" # Admin adjustment SYSTEM = "SYSTEM" # System credit (e.g., initial credit) + STORAGE = "STORAGE" # Metered storage overage (prepay pool) + SUBSCRIPTION = "SUBSCRIPTION" # AI budget credit from subscription plan class PeriodTypeEnum(str, Enum): @@ -48,14 +38,13 @@ class PeriodTypeEnum(str, Enum): YEAR = "YEAR" -class BillingAccount(BaseModel): +class BillingAccount(PowerOnModel): """Billing account for mandate or user-mandate combination.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) mandateId: str = Field(..., description="Foreign key to Mandate") - userId: Optional[str] = Field(None, description="Foreign key to User (only for PREPAY_USER)") - accountType: AccountTypeEnum = Field(..., description="Account type: MANDATE or USER") + userId: Optional[str] = Field(None, description="Foreign key to User (None = mandate pool account, set = user audit account)") balance: float = Field(default=0.0, description="Current balance in CHF") warningThreshold: float = Field(default=0.0, description="Warning threshold in CHF") lastWarningAt: Optional[datetime] = Field(None, description="Last warning sent timestamp") @@ -69,7 +58,6 @@ registerModelLabels( "id": {"en": "ID", "de": "ID"}, "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID"}, "userId": {"en": "User ID", "de": "Benutzer-ID"}, - "accountType": {"en": "Account Type", "de": "Kontotyp"}, "balance": {"en": "Balance (CHF)", "de": "Guthaben (CHF)"}, "warningThreshold": {"en": "Warning Threshold (CHF)", "de": "Warnschwelle (CHF)"}, "lastWarningAt": {"en": "Last Warning", "de": "Letzte Warnung"}, @@ -78,7 +66,7 @@ registerModelLabels( ) -class BillingTransaction(BaseModel): +class BillingTransaction(PowerOnModel): """Single billing transaction (credit, debit, adjustment).""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" @@ -129,30 +117,43 @@ registerModelLabels( class BillingSettings(BaseModel): - """Billing settings per mandate.""" + """Billing settings per mandate. Only PREPAY_MANDATE model.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) mandateId: str = Field(..., description="Foreign key to Mandate (UNIQUE)") - billingModel: BillingModelEnum = Field(..., description="Billing model") - - # Configuration - defaultUserCredit: float = Field( - default=0.0, - description="Automatic initial credit (CHF) for PREPAY_USER only when a user is newly added to the root mandate; other mandates use 0 on join.", - ) + warningThresholdPercent: float = Field(default=10.0, description="Warning threshold as percentage") # Stripe stripeCustomerId: Optional[str] = Field(None, description="Stripe Customer ID (cus_xxx) — one per mandate") - # Notifications (e.g. mandate owner / finance — also used when PREPAY_MANDATE pool is exhausted) + # Auto-Recharge for AI budget + autoRechargeEnabled: bool = Field(default=False, description="Auto-buy AI budget when low") + rechargeAmountCHF: float = Field(default=10.0, description="Amount per auto-recharge (CHF, prepaid via Stripe)") + rechargeMaxPerMonth: int = Field(default=3, description="Max auto-recharges per month") + rechargesThisMonth: int = Field(default=0, description="Counter: auto-recharges used this month") + monthResetAt: Optional[datetime] = Field(None, description="When rechargesThisMonth was last reset") + + # Notifications notifyEmails: List[str] = Field( default_factory=list, - description="Email addresses for billing alerts (mandate pool exhausted, warnings, etc.)", + description="Email addresses for billing alerts (pool exhausted, warnings, etc.)", ) notifyOnWarning: bool = Field(default=True, description="Send email when warning threshold is reached") + # Storage overage (high-watermark within subscription period; resets on new period) + storageHighWatermarkMB: float = Field( + default=0.0, description="Peak indexed data volume MB this billing period" + ) + storagePeriodStartAt: Optional[datetime] = Field( + None, description="Subscription billing period start used for storage reset" + ) + storageBilledUpToMB: float = Field( + default=0.0, + description="Overage MB already debited this period (above plan-included volume)", + ) + registerModelLabels( "BillingSettings", @@ -160,18 +161,22 @@ registerModelLabels( { "id": {"en": "ID", "de": "ID"}, "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID"}, - "billingModel": {"en": "Billing Model", "de": "Abrechnungsmodell"}, - "defaultUserCredit": { - "en": "Root start credit (CHF)", - "de": "Startguthaben nur Root-Mandant (CHF)", - }, "warningThresholdPercent": {"en": "Warning Threshold (%)", "de": "Warnschwelle (%)"}, "stripeCustomerId": {"en": "Stripe Customer ID", "de": "Stripe-Kunden-ID"}, + "autoRechargeEnabled": {"en": "Auto-Recharge", "de": "Auto-Nachladung"}, + "rechargeAmountCHF": {"en": "Recharge Amount (CHF)", "de": "Nachladebetrag (CHF)"}, + "rechargeMaxPerMonth": {"en": "Max Recharges/Month", "de": "Max. Nachladungen/Monat"}, "notifyEmails": { "en": "Billing notification emails (owner / admin)", - "de": "E-Mails für Billing-Alerts (Inhaber/Admin)", + "de": "E-Mails fuer Billing-Alerts (Inhaber/Admin)", }, "notifyOnWarning": {"en": "Notify on Warning", "de": "Bei Warnung benachrichtigen"}, + "storageHighWatermarkMB": {"en": "Storage peak (MB)", "de": "Speicher-Peak (MB)"}, + "storagePeriodStartAt": {"en": "Storage period start", "de": "Speicher-Periodenbeginn"}, + "storageBilledUpToMB": { + "en": "Storage billed overage (MB)", + "de": "Speicher abgerechneter Überhang (MB)", + }, }, ) @@ -238,7 +243,6 @@ class BillingBalanceResponse(BaseModel): """Response model for balance endpoint.""" mandateId: str mandateName: str - billingModel: BillingModelEnum balance: float currency: str = "CHF" warningThreshold: float @@ -269,20 +273,8 @@ class BillingCheckResult(BaseModel): reason: Optional[str] = None currentBalance: Optional[float] = None requiredAmount: Optional[float] = None - billingModel: Optional[BillingModelEnum] = None upgradeRequired: Optional[bool] = None subscriptionUiPath: Optional[str] = None userAction: Optional[str] = None -def parseBillingModelFromStoredValue(raw: Optional[str]) -> BillingModelEnum: - """Map DB string to enum. Legacy UNLIMITED / unknown values become PREPAY_MANDATE.""" - if raw is None or (isinstance(raw, str) and raw.strip() == ""): - return BillingModelEnum.PREPAY_MANDATE - s = str(raw).strip().upper() - if s == "UNLIMITED": - return BillingModelEnum.PREPAY_MANDATE - try: - return BillingModelEnum(raw) - except ValueError: - return BillingModelEnum.PREPAY_MANDATE diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 7002187a..7154e57e 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -5,12 +5,13 @@ from typing import List, Dict, Any, Optional from enum import Enum from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp import uuid -class ChatLog(BaseModel): +class ChatLog(PowerOnModel): """Log entries for chat workflows. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" @@ -56,7 +57,7 @@ registerModelLabels( ) -class ChatDocument(BaseModel): +class ChatDocument(PowerOnModel): """Documents attached to chat messages. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" @@ -163,7 +164,7 @@ registerModelLabels( ) -class ChatMessage(BaseModel): +class ChatMessage(PowerOnModel): """Messages in chat workflows. User-owned, no mandate context.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" @@ -260,7 +261,7 @@ registerModelLabels( ) -class ChatWorkflow(BaseModel): +class ChatWorkflow(PowerOnModel): """Chat workflow container. User-owned, no mandate context.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) featureInstanceId: Optional[str] = Field(None, description="Feature instance ID for multi-tenancy isolation", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) diff --git a/modules/datamodels/datamodelDataSource.py b/modules/datamodels/datamodelDataSource.py index baeac5ae..1d432041 100644 --- a/modules/datamodels/datamodelDataSource.py +++ b/modules/datamodels/datamodelDataSource.py @@ -8,12 +8,12 @@ Google Drive folder, FTP directory, etc.) for agent-accessible data containers. from typing import Dict, Any, Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp import uuid -class DataSource(BaseModel): +class DataSource(PowerOnModel): """Configured external data source linked to a UserConnection.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") connectionId: str = Field(description="FK to UserConnection") @@ -31,7 +31,21 @@ class DataSource(BaseModel): userId: str = Field(default="", description="Owner user ID") autoSync: bool = Field(default=False, description="Automatically sync on schedule") lastSynced: Optional[float] = Field(default=None, description="Last sync timestamp") - createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global", + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}}, + {"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}}, + {"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}}, + {"value": "global", "label": {"en": "Global", "de": "Global"}}, + ]} + ) + neutralize: bool = Field( + default=False, + description="Whether this data source should be neutralized before AI processing", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) registerModelLabels( @@ -49,7 +63,8 @@ registerModelLabels( "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, "autoSync": {"en": "Auto Sync", "de": "Auto-Sync", "fr": "Synchro auto"}, "lastSynced": {"en": "Last Synced", "de": "Letzter Sync", "fr": "Dernier sync"}, - "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, + "scope": {"en": "Scope", "de": "Sichtbarkeit"}, + "neutralize": {"en": "Neutralize", "de": "Neutralisieren"}, }, ) diff --git a/modules/datamodels/datamodelFeatureDataSource.py b/modules/datamodels/datamodelFeatureDataSource.py index 89b8b372..80ceb03c 100644 --- a/modules/datamodels/datamodelFeatureDataSource.py +++ b/modules/datamodels/datamodelFeatureDataSource.py @@ -8,12 +8,12 @@ so the agent can query structured feature data (e.g. TrusteePosition rows). from typing import Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp import uuid -class FeatureDataSource(BaseModel): +class FeatureDataSource(PowerOnModel): """A feature-instance table attached as data source in the AI workspace.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") featureInstanceId: str = Field(description="FK to FeatureInstance") @@ -24,7 +24,21 @@ class FeatureDataSource(BaseModel): mandateId: str = Field(default="", description="Mandate scope") userId: str = Field(default="", description="Owner user ID") workspaceInstanceId: str = Field(description="Workspace instance where this source is used") - createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global", + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}}, + {"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}}, + {"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}}, + {"value": "global", "label": {"en": "Global", "de": "Global"}}, + ]} + ) + neutralize: bool = Field( + default=False, + description="Whether this data source should be neutralized before AI processing", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) registerModelLabels( @@ -40,6 +54,5 @@ registerModelLabels( "mandateId": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, "userId": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"}, "workspaceInstanceId": {"en": "Workspace", "de": "Workspace", "fr": "Espace de travail"}, - "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, }, ) diff --git a/modules/datamodels/datamodelFeatures.py b/modules/datamodels/datamodelFeatures.py index 0a5dc441..3134a18e 100644 --- a/modules/datamodels/datamodelFeatures.py +++ b/modules/datamodels/datamodelFeatures.py @@ -5,11 +5,12 @@ import uuid from typing import Optional, Dict, Any from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.datamodels.datamodelUtils import TextMultilingual -class Feature(BaseModel): +class Feature(PowerOnModel): """ Feature-Definition (global, z.B. 'trustee', 'chatbot'). Features sind die verfügbaren Funktionalitäten der Plattform. @@ -40,7 +41,7 @@ registerModelLabels( ) -class FeatureInstance(BaseModel): +class FeatureInstance(PowerOnModel): """ Instanz eines Features in einem Mandanten. Ein Mandant kann mehrere Instanzen desselben Features haben. diff --git a/modules/datamodels/datamodelFileFolder.py b/modules/datamodels/datamodelFileFolder.py index b7a19915..23cd197b 100644 --- a/modules/datamodels/datamodelFileFolder.py +++ b/modules/datamodels/datamodelFileFolder.py @@ -4,18 +4,17 @@ from typing import Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp import uuid -class FileFolder(BaseModel): +class FileFolder(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) name: str = Field(description="Folder name", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) parentId: Optional[str] = Field(default=None, description="Parent folder ID (null = root)", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) mandateId: Optional[str] = Field(default=None, description="Mandate context", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) featureInstanceId: Optional[str] = Field(default=None, description="Feature instance context", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) registerModelLabels( @@ -27,6 +26,5 @@ registerModelLabels( "parentId": {"en": "Parent Folder", "fr": "Dossier parent"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, - "createdAt": {"en": "Created At", "fr": "Créé le"}, }, ) diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index afaad996..b8a44d2c 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -3,15 +3,14 @@ """File-related datamodels: FileItem, FilePreview, FileData.""" from typing import Dict, Any, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp import uuid import base64 -class FileItem(BaseModel): - model_config = ConfigDict(extra='allow') # Preserve system fields (_createdBy, _createdAt, etc.) +class FileItem(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: Optional[str] = Field(default="", description="ID of the mandate this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) featureInstanceId: Optional[str] = Field(default="", description="ID of the feature instance this file belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False, "frontend_fk_source": "/api/features/instances", "frontend_fk_display_field": "label"}) @@ -19,11 +18,25 @@ class FileItem(BaseModel): mimeType: str = Field(description="MIME type of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileHash: str = Field(description="Hash of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileSize: int = Field(description="Size of the file in bytes", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) - creationDate: float = Field(default_factory=getUtcTimestamp, description="Date when the file was created (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) tags: Optional[List[str]] = Field(default=None, description="Tags for categorization and search", json_schema_extra={"frontend_type": "tags", "frontend_readonly": False, "frontend_required": False}) folderId: Optional[str] = Field(default=None, description="ID of the parent folder", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) description: Optional[str] = Field(default=None, description="User-provided description of the file", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False}) status: Optional[str] = Field(default=None, description="Processing status: pending, extracted, embedding, indexed, failed", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global", + json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}}, + {"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}}, + {"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}}, + {"value": "global", "label": {"en": "Global", "de": "Global"}}, + ]} + ) + neutralize: bool = Field( + default=False, + description="Whether this file should be neutralized before AI processing", + json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} + ) registerModelLabels( "FileItem", @@ -36,11 +49,12 @@ registerModelLabels( "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, "tags": {"en": "Tags", "fr": "Tags"}, "folderId": {"en": "Folder ID", "fr": "ID du dossier"}, "description": {"en": "Description", "fr": "Description"}, "status": {"en": "Status", "fr": "Statut"}, + "scope": {"en": "Scope", "de": "Sichtbarkeit"}, + "neutralize": {"en": "Neutralize", "de": "Neutralisieren"}, }, ) @@ -71,7 +85,7 @@ registerModelLabels( }, ) -class FileData(BaseModel): +class FileData(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") data: str = Field(description="File data content") base64Encoded: bool = Field(description="Whether the data is base64 encoded") diff --git a/modules/datamodels/datamodelInvitation.py b/modules/datamodels/datamodelInvitation.py index 472318af..709e5021 100644 --- a/modules/datamodels/datamodelInvitation.py +++ b/modules/datamodels/datamodelInvitation.py @@ -9,11 +9,11 @@ import uuid import secrets from typing import Optional, List from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp -class Invitation(BaseModel): +class Invitation(PowerOnModel): """ Einladungs-Token für neue User. Ermöglicht Self-Service Onboarding zu Mandanten und Feature-Instanzen. @@ -56,15 +56,6 @@ class Invitation(BaseModel): description="Email address to send invitation link (optional)", json_schema_extra={"frontend_type": "email", "frontend_readonly": False, "frontend_required": False} ) - createdBy: str = Field( - description="User ID of the person who created the invitation", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True} - ) - createdAt: float = Field( - default_factory=getUtcTimestamp, - description="When the invitation was created (UTC timestamp)", - json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} - ) expiresAt: float = Field( description="When the invitation expires (UTC timestamp)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": True} @@ -121,8 +112,6 @@ registerModelLabels( "roleIds": {"en": "Roles", "de": "Rollen", "fr": "Rôles"}, "targetUsername": {"en": "Target Username", "de": "Ziel-Benutzername", "fr": "Nom d'utilisateur cible"}, "email": {"en": "Email (optional)", "de": "E-Mail (optional)", "fr": "Email (optionnel)"}, - "createdBy": {"en": "Created By", "de": "Erstellt von", "fr": "Créé par"}, - "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, "expiresAt": {"en": "Expires At", "de": "Gültig bis", "fr": "Expire le"}, "usedBy": {"en": "Used By", "de": "Verwendet von", "fr": "Utilisé par"}, "usedAt": {"en": "Used At", "de": "Verwendet am", "fr": "Utilisé le"}, diff --git a/modules/datamodels/datamodelKnowledge.py b/modules/datamodels/datamodelKnowledge.py index d03e9d5a..7ac12c15 100644 --- a/modules/datamodels/datamodelKnowledge.py +++ b/modules/datamodels/datamodelKnowledge.py @@ -3,8 +3,10 @@ """Knowledge Store data models: FileContentIndex, ContentChunk, WorkflowMemory. These models support the 3-tier RAG architecture: -- Shared Layer: mandateId-scoped, isShared=True -- Instance Layer: userId + featureInstanceId-scoped +- Personal Layer: scope=personal, userId-scoped +- Instance Layer: scope=featureInstance, featureInstanceId-scoped +- Mandate Layer: scope=mandate, mandateId-scoped (visible to all mandate users) +- Global Layer: scope=global (sysAdmin only) - Workflow Layer: workflowId-scoped (WorkflowMemory) Vector fields use json_schema_extra={"db_type": "vector(1536)"} for pgvector. @@ -12,19 +14,19 @@ Vector fields use json_schema_extra={"db_type": "vector(1536)"} for pgvector. from typing import Dict, Any, List, Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp import uuid -class FileContentIndex(BaseModel): +class FileContentIndex(PowerOnModel): """Structural index of a file's content objects. Created without AI. - Lives in the Instance Layer; optionally promoted to Shared Layer via isShared.""" + Scope is mirrored from FileItem (poweron_management) at indexing time.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key (typically = fileId)") userId: str = Field(description="Owner user ID") featureInstanceId: str = Field(default="", description="Feature instance scope") mandateId: str = Field(default="", description="Mandate scope") - isShared: bool = Field(default=False, description="Visible in Shared Layer for all mandate users") fileName: str = Field(description="Original file name") mimeType: str = Field(description="MIME type of the file") containerPath: Optional[str] = Field(default=None, description="Path within a container (e.g. 'archive.zip/folder/report.pdf')") @@ -34,6 +36,18 @@ class FileContentIndex(BaseModel): objectSummary: List[Dict[str, Any]] = Field(default_factory=list, description="Compact summary per content object") extractedAt: float = Field(default_factory=getUtcTimestamp, description="Extraction timestamp") status: str = Field(default="pending", description="Processing status: pending, extracted, embedding, indexed, failed") + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global", + ) + neutralizationStatus: Optional[str] = Field( + default=None, + description="Neutralization status: completed, failed, skipped, None = not required", + ) + isNeutralized: bool = Field( + default=False, + description="True if content was neutralized before indexing", + ) registerModelLabels( @@ -44,7 +58,6 @@ registerModelLabels( "userId": {"en": "User ID", "fr": "ID utilisateur"}, "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "isShared": {"en": "Shared", "fr": "Partagé"}, "fileName": {"en": "File Name", "fr": "Nom de fichier"}, "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, "containerPath": {"en": "Container Path", "fr": "Chemin du conteneur"}, @@ -54,11 +67,14 @@ registerModelLabels( "objectSummary": {"en": "Object Summary", "fr": "Résumé des objets"}, "extractedAt": {"en": "Extracted At", "fr": "Extrait le"}, "status": {"en": "Status", "fr": "Statut"}, + "scope": {"en": "Scope", "de": "Sichtbarkeit"}, + "neutralizationStatus": {"en": "Neutralization Status", "de": "Neutralisierungsstatus"}, + "isNeutralized": {"en": "Is Neutralized", "de": "Neutralisiert"}, }, ) -class ContentChunk(BaseModel): +class ContentChunk(PowerOnModel): """Persisted content chunk with embedding vector. Reusable across workflows. Scalar content object (or chunk thereof) with pgvector embedding.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") @@ -96,7 +112,7 @@ registerModelLabels( ) -class RoundMemory(BaseModel): +class RoundMemory(PowerOnModel): """Persistent per-round memory for agent tool results, file refs, and decisions. Stored after each agent round so that RAG can retrieve relevant context @@ -120,7 +136,6 @@ class RoundMemory(BaseModel): description="Embedding of summary for semantic retrieval", json_schema_extra={"db_type": "vector(1536)"}, ) - createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") registerModelLabels( @@ -136,12 +151,11 @@ registerModelLabels( "fullData": {"en": "Full Data", "fr": "Données complètes"}, "fileIds": {"en": "File IDs", "fr": "IDs de fichier"}, "embedding": {"en": "Embedding", "fr": "Vecteur d'embedding"}, - "createdAt": {"en": "Created At", "fr": "Créé le"}, }, ) -class WorkflowMemory(BaseModel): +class WorkflowMemory(PowerOnModel): """Workflow-scoped key-value cache for entities and facts. Extracted during agent rounds, persisted for cross-round and cross-workflow reuse.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") @@ -151,7 +165,6 @@ class WorkflowMemory(BaseModel): key: str = Field(description="Key identifier (e.g. 'entity:companyName')") value: str = Field(description="Extracted value") source: str = Field(default="extraction", description="Origin: extraction, tool, conversation, summary") - createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") embedding: Optional[List[float]] = Field( default=None, description="Optional embedding for semantic lookup", json_schema_extra={"db_type": "vector(1536)"} @@ -169,7 +182,6 @@ registerModelLabels( "key": {"en": "Key", "fr": "Clé"}, "value": {"en": "Value", "fr": "Valeur"}, "source": {"en": "Source", "fr": "Source"}, - "createdAt": {"en": "Created At", "fr": "Créé le"}, "embedding": {"en": "Embedding", "fr": "Vecteur d'embedding"}, }, ) diff --git a/modules/datamodels/datamodelMembership.py b/modules/datamodels/datamodelMembership.py index 5e8b8814..ce753d15 100644 --- a/modules/datamodels/datamodelMembership.py +++ b/modules/datamodels/datamodelMembership.py @@ -9,10 +9,11 @@ Rollen werden über Junction Tables verknüpft für saubere CASCADE DELETE. import uuid from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -class UserMandate(BaseModel): +class UserMandate(PowerOnModel): """ User-Mitgliedschaft in einem Mandanten. Kein User gehört direkt zu einem Mandanten - Zugehörigkeit wird über dieses Model gesteuert. @@ -50,7 +51,7 @@ registerModelLabels( ) -class FeatureAccess(BaseModel): +class FeatureAccess(PowerOnModel): """ User-Zugriff auf eine Feature-Instanz. Definiert welche User auf welche Feature-Instanzen zugreifen können. @@ -88,7 +89,7 @@ registerModelLabels( ) -class UserMandateRole(BaseModel): +class UserMandateRole(PowerOnModel): """ Junction Table: UserMandate zu Role. Ermöglicht CASCADE DELETE auf Datenbankebene. @@ -119,7 +120,7 @@ registerModelLabels( ) -class FeatureAccessRole(BaseModel): +class FeatureAccessRole(PowerOnModel): """ Junction Table: FeatureAccess zu Role. Ermöglicht CASCADE DELETE auf Datenbankebene. diff --git a/modules/datamodels/datamodelMessaging.py b/modules/datamodels/datamodelMessaging.py index 1c2206b7..ebacc9d4 100644 --- a/modules/datamodels/datamodelMessaging.py +++ b/modules/datamodels/datamodelMessaging.py @@ -6,8 +6,8 @@ import uuid from typing import Optional from enum import Enum from pydantic import BaseModel, Field, ConfigDict +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp class MessagingChannel(str, Enum): @@ -26,7 +26,7 @@ class DeliveryStatus(str, Enum): FAILED = "failed" -class MessagingSubscription(BaseModel): +class MessagingSubscription(PowerOnModel): """Data model for messaging subscriptions""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -64,26 +64,6 @@ class MessagingSubscription(BaseModel): description="Whether the subscription is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} ) - creationDate: float = Field( - default_factory=getUtcTimestamp, - description="When the subscription was created (UTC timestamp in seconds)", - json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} - ) - lastModified: float = Field( - default_factory=getUtcTimestamp, - description="When the subscription was last modified (UTC timestamp in seconds)", - json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} - ) - createdBy: Optional[str] = Field( - default=None, - description="User ID who created the subscription", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} - ) - modifiedBy: Optional[str] = Field( - default=None, - description="User ID who last modified the subscription", - json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} - ) model_config = ConfigDict(use_enum_values=True) @@ -100,10 +80,6 @@ registerModelLabels( "description": {"en": "Description", "fr": "Description"}, "isSystemSubscription": {"en": "System Subscription", "fr": "Abonnement système"}, "enabled": {"en": "Enabled", "fr": "Activé"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, - "lastModified": {"en": "Last Modified", "fr": "Dernière modification"}, - "createdBy": {"en": "Created By", "fr": "Créé par"}, - "modifiedBy": {"en": "Modified By", "fr": "Modifié par"}, }, ) @@ -155,16 +131,6 @@ class MessagingSubscriptionRegistration(BaseModel): description="Whether this registration is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False} ) - creationDate: float = Field( - default_factory=getUtcTimestamp, - description="When the registration was created (UTC timestamp in seconds)", - json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} - ) - lastModified: float = Field( - default_factory=getUtcTimestamp, - description="When the registration was last modified (UTC timestamp in seconds)", - json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} - ) model_config = ConfigDict(use_enum_values=True) @@ -181,8 +147,6 @@ registerModelLabels( "channel": {"en": "Channel", "fr": "Canal"}, "channelConfig": {"en": "Channel Config", "fr": "Configuration du canal"}, "enabled": {"en": "Enabled", "fr": "Activé"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, - "lastModified": {"en": "Last Modified", "fr": "Dernière modification"}, }, ) @@ -248,11 +212,6 @@ class MessagingDelivery(BaseModel): description="When the delivery was sent (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} ) - creationDate: float = Field( - default_factory=getUtcTimestamp, - description="When the delivery record was created (UTC timestamp in seconds)", - json_schema_extra={"frontend_type": "datetime", "frontend_readonly": True, "frontend_required": False} - ) model_config = ConfigDict(use_enum_values=True) @@ -270,7 +229,6 @@ registerModelLabels( "status": {"en": "Status", "fr": "Statut"}, "errorMessage": {"en": "Error Message", "fr": "Message d'erreur"}, "sentAt": {"en": "Sent At", "fr": "Envoyé le"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, }, ) @@ -349,4 +307,3 @@ class MessagingSubscriptionExecutionResult(BaseModel): description="Error message if execution failed", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": True, "frontend_required": False} ) - model_config = ConfigDict(extra="allow") # Allow additional fields for custom results diff --git a/modules/datamodels/datamodelNotification.py b/modules/datamodels/datamodelNotification.py index b1475767..f5af0f55 100644 --- a/modules/datamodels/datamodelNotification.py +++ b/modules/datamodels/datamodelNotification.py @@ -9,8 +9,8 @@ import uuid from typing import Optional, List from enum import Enum from pydantic import BaseModel, Field, ConfigDict +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp class NotificationType(str, Enum): @@ -43,7 +43,7 @@ class NotificationAction(BaseModel): ) -class UserNotification(BaseModel): +class UserNotification(PowerOnModel): """ In-app notification for a user. Supports actionable notifications with accept/decline buttons. @@ -137,11 +137,6 @@ class UserNotification(BaseModel): ) # Timestamps - createdAt: float = Field( - default_factory=getUtcTimestamp, - description="When the notification was created (UTC timestamp)", - json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} - ) readAt: Optional[float] = Field( default=None, description="When the notification was read (UTC timestamp)", @@ -177,7 +172,6 @@ registerModelLabels( "actions": {"en": "Actions", "de": "Aktionen", "fr": "Actions"}, "actionTaken": {"en": "Action Taken", "de": "Durchgeführte Aktion", "fr": "Action effectuée"}, "actionResult": {"en": "Action Result", "de": "Aktions-Ergebnis", "fr": "Résultat de l'action"}, - "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, "readAt": {"en": "Read At", "de": "Gelesen am", "fr": "Lu le"}, "actionedAt": {"en": "Actioned At", "de": "Bearbeitet am", "fr": "Traité le"}, "expiresAt": {"en": "Expires At", "de": "Gültig bis", "fr": "Expire le"}, diff --git a/modules/datamodels/datamodelRbac.py b/modules/datamodels/datamodelRbac.py index 978c3be6..b9e0cb91 100644 --- a/modules/datamodels/datamodelRbac.py +++ b/modules/datamodels/datamodelRbac.py @@ -13,6 +13,7 @@ import uuid from typing import Optional from enum import Enum from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.datamodels.datamodelUtils import TextMultilingual from modules.datamodels.datamodelUam import AccessLevel @@ -25,7 +26,7 @@ class AccessRuleContext(str, Enum): RESOURCE = "RESOURCE" # System resources (AI models, actions, etc.) -class Role(BaseModel): +class Role(PowerOnModel): """ Data model for RBAC roles. @@ -90,7 +91,7 @@ registerModelLabels( ) -class AccessRule(BaseModel): +class AccessRule(PowerOnModel): """ Data model for access control rules. diff --git a/modules/datamodels/datamodelSecurity.py b/modules/datamodels/datamodelSecurity.py index 5caafe1b..dc8c26e6 100644 --- a/modules/datamodels/datamodelSecurity.py +++ b/modules/datamodels/datamodelSecurity.py @@ -11,6 +11,7 @@ Multi-Tenant Design: from typing import Optional, Any from pydantic import BaseModel, Field, ConfigDict, model_validator +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp from .datamodelUam import AuthAuthority @@ -30,7 +31,7 @@ class TokenPurpose(str, Enum): DATA_CONNECTION = "dataConnection" -class Token(BaseModel): +class Token(PowerOnModel): """ Authentication Token model. @@ -55,9 +56,6 @@ class Token(BaseModel): description="When the token expires (UTC timestamp in seconds)" ) tokenRefresh: Optional[str] = None - createdAt: Optional[float] = Field( - None, description="When the token was created (UTC timestamp in seconds)" - ) status: TokenStatus = Field( default=TokenStatus.ACTIVE, description="Token status: active/revoked" ) @@ -106,7 +104,6 @@ registerModelLabels( "tokenType": {"en": "Token Type", "de": "Token-Typ", "fr": "Type de jeton"}, "expiresAt": {"en": "Expires At", "de": "Läuft ab am", "fr": "Expire le"}, "tokenRefresh": {"en": "Refresh Token", "de": "Refresh-Token", "fr": "Jeton de rafraîchissement"}, - "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, "status": {"en": "Status", "de": "Status", "fr": "Statut"}, "revokedAt": {"en": "Revoked At", "de": "Widerrufen am", "fr": "Révoqué le"}, "revokedBy": {"en": "Revoked By", "de": "Widerrufen von", "fr": "Révoqué par"}, @@ -116,7 +113,7 @@ registerModelLabels( ) -class AuthEvent(BaseModel): +class AuthEvent(PowerOnModel): """Authentication event for audit logging.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the auth event", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user this event belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) diff --git a/modules/datamodels/datamodelSubscription.py b/modules/datamodels/datamodelSubscription.py index 1c1435d8..8fcf10f2 100644 --- a/modules/datamodels/datamodelSubscription.py +++ b/modules/datamodels/datamodelSubscription.py @@ -10,6 +10,7 @@ from typing import Dict, List, Optional from enum import Enum from datetime import datetime, timezone from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels import uuid @@ -30,6 +31,7 @@ OPERATIVE_STATUSES = {SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.TRIA ALLOWED_TRANSITIONS = { (SubscriptionStatusEnum.PENDING, SubscriptionStatusEnum.ACTIVE), + (SubscriptionStatusEnum.PENDING, SubscriptionStatusEnum.TRIALING), (SubscriptionStatusEnum.PENDING, SubscriptionStatusEnum.SCHEDULED), (SubscriptionStatusEnum.PENDING, SubscriptionStatusEnum.EXPIRED), (SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE), @@ -70,6 +72,8 @@ class SubscriptionPlan(BaseModel): maxUsers: Optional[int] = Field(None, description="Hard cap on active users (None = unlimited)") maxFeatureInstances: Optional[int] = Field(None, description="Hard cap on active feature instances (None = unlimited)") trialDays: Optional[int] = Field(None, description="Trial duration in days (only for trial plans)") + maxDataVolumeMB: Optional[int] = Field(None, description="Soft-limit for data volume in MB per mandate (None = unlimited)") + budgetAiCHF: float = Field(default=0.0, description="AI budget (CHF) included in subscription price per billing period") successorPlanKey: Optional[str] = Field(None, description="Plan to transition to when trial ends") @@ -84,6 +88,8 @@ registerModelLabels( "pricePerFeatureInstanceCHF": {"en": "Price per Instance (CHF)", "de": "Preis pro Instanz (CHF)"}, "maxUsers": {"en": "Max Users", "de": "Max. Benutzer", "fr": "Max. utilisateurs"}, "maxFeatureInstances": {"en": "Max Instances", "de": "Max. Instanzen", "fr": "Max. instances"}, + "maxDataVolumeMB": {"en": "Data Volume (MB)", "de": "Datenvolumen (MB)"}, + "budgetAiCHF": {"en": "AI Budget (CHF)", "de": "AI-Budget (CHF)"}, }, ) @@ -122,7 +128,7 @@ registerModelLabels( # Instance: MandateSubscription # ============================================================================ -class MandateSubscription(BaseModel): +class MandateSubscription(PowerOnModel): """A subscription instance bound to a specific mandate. See wiki/concepts/Subscription-State-Machine.md for state transitions.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") @@ -182,20 +188,24 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = { autoRenew=False, maxUsers=None, maxFeatureInstances=None, + maxDataVolumeMB=None, + budgetAiCHF=0.0, ), "TRIAL_7D": SubscriptionPlan( planKey="TRIAL_7D", selectableByUser=False, title={"en": "Free Trial (7 days)", "de": "Gratis-Testphase (7 Tage)", "fr": "Essai gratuit (7 jours)"}, description={ - "en": "Try the platform for 7 days — 1 user, up to 3 feature instances.", - "de": "Plattform 7 Tage testen — 1 User, bis zu 3 Feature-Instanzen.", + "en": "Try the platform for 7 days — 1 user, up to 3 feature instances, 5 CHF AI budget included.", + "de": "Plattform 7 Tage testen — 1 User, bis zu 3 Feature-Instanzen, 5 CHF AI-Budget inklusive.", }, billingPeriod=BillingPeriodEnum.NONE, autoRenew=False, maxUsers=1, maxFeatureInstances=3, trialDays=7, + maxDataVolumeMB=500, + budgetAiCHF=5.0, successorPlanKey="STANDARD_MONTHLY", ), "STANDARD_MONTHLY": SubscriptionPlan( @@ -203,24 +213,28 @@ BUILTIN_PLANS: Dict[str, SubscriptionPlan] = { selectableByUser=True, title={"en": "Standard (Monthly)", "de": "Standard (Monatlich)", "fr": "Standard (Mensuel)"}, description={ - "en": "Usage-based billing per active user and feature instance, billed monthly.", - "de": "Nutzungsbasierte Abrechnung pro aktivem User und Feature-Instanz, monatlich.", + "en": "Usage-based billing per active user and feature instance, billed monthly. Includes 10 CHF AI budget.", + "de": "Nutzungsbasierte Abrechnung pro aktivem User und Feature-Instanz, monatlich. Inkl. 10 CHF AI-Budget.", }, billingPeriod=BillingPeriodEnum.MONTHLY, pricePerUserCHF=90.0, pricePerFeatureInstanceCHF=150.0, + maxDataVolumeMB=1024, + budgetAiCHF=10.0, ), "STANDARD_YEARLY": SubscriptionPlan( planKey="STANDARD_YEARLY", selectableByUser=True, title={"en": "Standard (Yearly)", "de": "Standard (Jährlich)", "fr": "Standard (Annuel)"}, description={ - "en": "Usage-based billing per active user and feature instance, billed yearly.", - "de": "Nutzungsbasierte Abrechnung pro aktivem User und Feature-Instanz, jährlich.", + "en": "Usage-based billing per active user and feature instance, billed yearly. Includes 120 CHF AI budget.", + "de": "Nutzungsbasierte Abrechnung pro aktivem User und Feature-Instanz, jährlich. Inkl. 120 CHF AI-Budget.", }, billingPeriod=BillingPeriodEnum.YEARLY, pricePerUserCHF=1080.0, pricePerFeatureInstanceCHF=1800.0, + maxDataVolumeMB=1024, + budgetAiCHF=120.0, ), } diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index 78858c8d..35e9ec7c 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -10,9 +10,10 @@ Multi-Tenant Design: """ import uuid -from typing import Optional, List +from typing import Optional, List, Dict, Any from enum import Enum from pydantic import BaseModel, Field, EmailStr, field_validator, computed_field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp @@ -60,7 +61,7 @@ class UserPermissions(BaseModel): ) -class Mandate(BaseModel): +class Mandate(PowerOnModel): """ Mandate (Mandant/Tenant) model. Ein Mandant ist ein isolierter Bereich für Daten und Berechtigungen. @@ -89,6 +90,11 @@ class Mandate(BaseModel): description="Whether this is a system mandate (e.g. root mandate). Cannot be deleted.", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": True, "frontend_required": False} ) + deletedAt: Optional[float] = Field( + default=None, + description="Timestamp when the mandate was soft-deleted. After 30 days, hard-delete is triggered.", + json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False} + ) @field_validator('isSystem', mode='before') @classmethod @@ -98,7 +104,6 @@ class Mandate(BaseModel): return False return v - registerModelLabels( "Mandate", {"en": "Mandate", "de": "Mandant", "fr": "Mandat"}, @@ -108,11 +113,12 @@ registerModelLabels( "label": {"en": "Label", "de": "Label", "fr": "Libellé"}, "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, "isSystem": {"en": "System Mandate", "de": "System-Mandant", "fr": "Mandat système"}, + "deletedAt": {"en": "Deleted at", "de": "Gelöscht am", "fr": "Supprimé le"}, }, ) -class UserConnection(BaseModel): +class UserConnection(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the connection", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="ID of the user this connection belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) authority: AuthAuthority = Field(description="Authentication authority", json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "/api/connections/authorities/options"}) @@ -174,7 +180,7 @@ registerModelLabels( ) -class User(BaseModel): +class User(PowerOnModel): """ User model. @@ -261,6 +267,11 @@ class User(BaseModel): description="Primary authentication authority", json_schema_extra={"frontend_type": "select", "frontend_readonly": True, "frontend_required": False, "frontend_options": "/api/connections/authorities/options"} ) + roleLabels: List[str] = Field( + default_factory=list, + description="Role labels (from DB or enriched when loading users)", + json_schema_extra={"frontend_type": "multiselect", "frontend_readonly": True, "frontend_visible": False, "frontend_required": False}, + ) registerModelLabels( @@ -275,6 +286,7 @@ registerModelLabels( "enabled": {"en": "Enabled", "de": "Aktiviert", "fr": "Activé"}, "isSysAdmin": {"en": "System Admin", "de": "System-Admin", "fr": "Admin système"}, "authenticationAuthority": {"en": "Auth Authority", "de": "Authentifizierung", "fr": "Autorité d'authentification"}, + "roleLabels": {"en": "Role Labels", "de": "Rollen-Labels", "fr": "Libellés de rôles"}, }, ) @@ -295,3 +307,65 @@ registerModelLabels( "resetTokenExpires": {"en": "Reset Token Expires", "de": "Token läuft ab", "fr": "Expiration du jeton"}, }, ) + + +def _normalizeTtsVoiceMap(value: Any) -> Optional[Dict[str, str]]: + """ + Coerce ttsVoiceMap payloads to Dict[str, str]. + + UI/clients may send per-locale objects like {"voiceName": "de-DE-Chirp3-HD-Achird"}; + storage and model field type are locale -> voice id string. + """ + if value is None: + return None + if not isinstance(value, dict): + return None + out: Dict[str, str] = {} + for rawKey, rawVal in value.items(): + key = str(rawKey) + if rawVal is None: + continue + if isinstance(rawVal, str): + out[key] = rawVal + elif isinstance(rawVal, dict): + vn = rawVal.get("voiceName") + if vn is not None and str(vn).strip() != "": + out[key] = str(vn).strip() + else: + out[key] = str(rawVal) + return out if out else None + + +class UserVoicePreferences(PowerOnModel): + """User-level voice/language preferences, shared across all features.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + userId: str = Field(description="User ID") + mandateId: Optional[str] = Field(default=None, description="Mandate scope (None = global for user)") + sttLanguage: str = Field(default="de-DE", description="Speech-to-text language code") + ttsLanguage: str = Field(default="de-DE", description="Text-to-speech language code") + ttsVoice: Optional[str] = Field(default=None, description="Preferred TTS voice identifier") + ttsVoiceMap: Optional[Dict[str, str]] = Field(default=None, description="Language-to-voice mapping") + translationSourceLanguage: Optional[str] = Field(default=None, description="Source language for translations") + translationTargetLanguage: Optional[str] = Field(default=None, description="Target language for translations") + + @field_validator("ttsVoiceMap", mode="before") + @classmethod + def _validateTtsVoiceMap(cls, value: Any) -> Optional[Dict[str, str]]: + return _normalizeTtsVoiceMap(value) + + +registerModelLabels( + "UserVoicePreferences", + {"en": "Voice Preferences", "de": "Spracheinstellungen", "fr": "Préférences vocales"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, + "sttLanguage": {"en": "STT Language", "de": "STT-Sprache", "fr": "Langue STT"}, + "ttsLanguage": {"en": "TTS Language", "de": "TTS-Sprache", "fr": "Langue TTS"}, + "ttsVoice": {"en": "TTS Voice", "de": "TTS-Stimme", "fr": "Voix TTS"}, + "ttsVoiceMap": {"en": "Voice Map", "de": "Stimmen-Zuordnung", "fr": "Carte des voix"}, + "translationSourceLanguage": {"en": "Translation Source", "de": "Übersetzung Quelle", "fr": "Langue source"}, + "translationTargetLanguage": {"en": "Translation Target", "de": "Übersetzung Ziel", "fr": "Langue cible"}, + }, +) diff --git a/modules/datamodels/datamodelUtils.py b/modules/datamodels/datamodelUtils.py index 614d6592..1088cb31 100644 --- a/modules/datamodels/datamodelUtils.py +++ b/modules/datamodels/datamodelUtils.py @@ -3,13 +3,13 @@ """Utility datamodels: Prompt, TextMultilingual.""" from typing import Dict, Optional -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, Field, field_validator +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels import uuid -class Prompt(BaseModel): - model_config = ConfigDict(extra='allow') # Preserve system fields (_createdBy, _createdAt, etc.) +class Prompt(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(default="", description="ID of the mandate this prompt belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) isSystem: bool = Field(default=False, description="System prompt visible to all users (read-only for non-SysAdmin)", json_schema_extra={"frontend_type": "boolean", "frontend_readonly": True, "frontend_required": False}) diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py index 565c7677..c3a622ac 100644 --- a/modules/datamodels/datamodelVoice.py +++ b/modules/datamodels/datamodelVoice.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""Voice settings datamodel — re-exported from workspace feature for backward compatibility.""" +"""Voice settings datamodel — re-exported from UAM for central voice preferences.""" -from modules.features.workspace.datamodelFeatureWorkspace import VoiceSettings +from modules.datamodels.datamodelUam import UserVoicePreferences -__all__ = ["VoiceSettings"] +__all__ = ["UserVoicePreferences"] diff --git a/modules/features/automation/datamodelFeatureAutomation.py b/modules/features/automation/datamodelFeatureAutomation.py index 732f3163..8ea4a300 100644 --- a/modules/features/automation/datamodelFeatureAutomation.py +++ b/modules/features/automation/datamodelFeatureAutomation.py @@ -4,6 +4,7 @@ from typing import List, Dict, Any, Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.datamodels.datamodelUtils import TextMultilingual import uuid @@ -48,7 +49,7 @@ registerModelLabels( ) -class AutomationTemplate(BaseModel): +class AutomationTemplate(PowerOnModel): """Automation-Vorlage ohne scharfe Placeholder-Werte (DB-persistiert). System-Templates (isSystem=True): Nur durch SysAdmin aenderbar. Alle User koennen lesen. @@ -82,9 +83,6 @@ class AutomationTemplate(BaseModel): description="Feature instance ID (null for system templates, set for instance-scoped templates)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False} ) - # System fields (_createdAt, _createdBy, etc.) werden automatisch vom DB-Connector gesetzt - - registerModelLabels( "AutomationTemplate", {"en": "Automation Template", "ge": "Automation-Vorlage", "fr": "Modèle d'automatisation"}, diff --git a/modules/features/automation/interfaceFeatureAutomation.py b/modules/features/automation/interfaceFeatureAutomation.py index 4091bc28..a4f90a51 100644 --- a/modules/features/automation/interfaceFeatureAutomation.py +++ b/modules/features/automation/interfaceFeatureAutomation.py @@ -22,6 +22,13 @@ from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) + +def _automationDefinitionPayload(data: Dict[str, Any]) -> Dict[str, Any]: + """Strip connector/enrichment keys; only fields defined on AutomationDefinition.""" + allowed = AutomationDefinition.model_fields.keys() + return {k: v for k, v in (data or {}).items() if k in allowed} + + # Singleton factory for Automation instances _automationInterfaces = {} @@ -100,7 +107,7 @@ class AutomationObjects: if recordId: record = self.db.getRecordset(model, recordFilter={"id": recordId}) if record: - return record[0].get("_createdBy") == self.userId + return record[0].get("sysCreatedBy") == self.userId else: return False # Record not found = no access return True # No recordId needed (e.g., for CREATE) @@ -130,7 +137,7 @@ class AutomationObjects: featureInstanceIds = set() for automation in automations: - createdBy = automation.get("_createdBy") + createdBy = automation.get("sysCreatedBy") if createdBy: userIds.add(createdBy) @@ -186,8 +193,8 @@ class AutomationObjects: # Enrich each automation with the fetched data # SECURITY: Never show a fallback name — if lookup fails, show empty string for automation in automations: - createdBy = automation.get("_createdBy") - automation["_createdByUserName"] = usersMap.get(createdBy, "") if createdBy else "" + createdBy = automation.get("sysCreatedBy") + automation["sysCreatedByUserName"] = usersMap.get(createdBy, "") if createdBy else "" mandateId = automation.get("mandateId") automation["mandateName"] = mandatesMap.get(mandateId, "") if mandateId else "" @@ -295,7 +302,7 @@ class AutomationObjects: Args: automationId: ID of the automation to get - includeSystemFields: If True, returns raw dict with system fields (_createdBy, etc). + includeSystemFields: If True, returns raw dict with system fields (sysCreatedBy, etc). If False (default), returns Pydantic model without system fields. """ try: @@ -330,7 +337,7 @@ class AutomationObjects: return AutomationWithSystemFields(automation) # Clean metadata fields and return Pydantic model - cleanedRecord = {k: v for k, v in automation.items() if not k.startswith("_")} + cleanedRecord = _automationDefinitionPayload(automation) return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error getting automation definition: {str(e)}") @@ -365,7 +372,7 @@ class AutomationObjects: # Ensure database connector has correct userId context if not self.userId: - logger.error(f"createAutomationDefinition: userId is not set! Cannot set _createdBy. currentUser={self.currentUser}") + logger.error(f"createAutomationDefinition: userId is not set! Cannot set sysCreatedBy. currentUser={self.currentUser}") elif hasattr(self.db, 'updateContext'): try: self.db.updateContext(self.userId) @@ -386,7 +393,7 @@ class AutomationObjects: self._notifyAutomationChanged() # Clean metadata fields and return Pydantic model - cleanedRecord = {k: v for k, v in createdAutomation.items() if not k.startswith("_")} + cleanedRecord = _automationDefinitionPayload(createdAutomation) return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error creating automation definition: {str(e)}") @@ -446,7 +453,7 @@ class AutomationObjects: self._notifyAutomationChanged() # Clean metadata fields and return Pydantic model - cleanedRecord = {k: v for k, v in updatedAutomation.items() if not k.startswith("_")} + cleanedRecord = _automationDefinitionPayload(updatedAutomation) return AutomationDefinition(**cleanedRecord) except Exception as e: logger.error(f"Error updating automation definition: {str(e)}") @@ -561,7 +568,7 @@ class AutomationObjects: # Collect unique user IDs userIds = set() for template in templates: - createdBy = template.get("_createdBy") + createdBy = template.get("sysCreatedBy") if createdBy: userIds.add(createdBy) @@ -585,8 +592,8 @@ class AutomationObjects: # Apply to templates — SECURITY: no fallback, empty if not found for template in templates: - createdBy = template.get("_createdBy") - template["_createdByUserName"] = userNameMap.get(createdBy, "") if createdBy else "" + createdBy = template.get("sysCreatedBy") + template["sysCreatedByUserName"] = userNameMap.get(createdBy, "") if createdBy else "" except Exception as e: logger.warning(f"Could not enrich templates with user names: {e}") diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py index 4bb30f7f..d56804fd 100644 --- a/modules/features/automation/mainAutomation.py +++ b/modules/features/automation/mainAutomation.py @@ -227,7 +227,7 @@ def getFeatureDefinition() -> Dict[str, Any]: "code": FEATURE_CODE, "label": FEATURE_LABEL, "icon": FEATURE_ICON, - "autoCreateInstance": True, # Automatically create instance in root mandate during bootstrap + "autoCreateInstance": False, } diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 48f53eea..c6343b25 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -77,8 +77,8 @@ def get_automations( # If pagination was requested, result is PaginatedResult # If no pagination, result is List[Dict] - # Note: Using JSONResponse to bypass Pydantic validation which would filter out _createdBy - # The enriched fields (_createdByUserName, mandateName) are not in the Pydantic model + # Note: Using JSONResponse to bypass Pydantic validation which would filter out sysCreatedBy + # The enriched fields (sysCreatedByUserName, mandateName) are not in the Pydantic model from fastapi.responses import JSONResponse if paginationParams: diff --git a/modules/features/automation2/datamodelFeatureAutomation2.py b/modules/features/automation2/datamodelFeatureAutomation2.py index 4ccab460..97b33754 100644 --- a/modules/features/automation2/datamodelFeatureAutomation2.py +++ b/modules/features/automation2/datamodelFeatureAutomation2.py @@ -4,6 +4,7 @@ from typing import Dict, Any, List, Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels import uuid @@ -58,7 +59,7 @@ registerModelLabels( ) -class Automation2WorkflowRun(BaseModel): +class Automation2WorkflowRun(PowerOnModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key", @@ -104,7 +105,7 @@ registerModelLabels( ) -class Automation2HumanTask(BaseModel): +class Automation2HumanTask(PowerOnModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key", diff --git a/modules/features/automation2/mainAutomation2.py b/modules/features/automation2/mainAutomation2.py index 4f52e158..80c8f854 100644 --- a/modules/features/automation2/mainAutomation2.py +++ b/modules/features/automation2/mainAutomation2.py @@ -62,12 +62,25 @@ RESOURCE_OBJECTS = [ ] TEMPLATE_ROLES = [ + { + "roleLabel": "automation2-viewer", + "description": { + "en": "Automation2 Viewer - View workflows (read-only)", + "de": "Automation2 Betrachter - Workflows ansehen (nur lesen)", + "fr": "Visualiseur Automation2 - Consulter les workflows (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.automation2.workflows", "view": True}, + {"context": "UI", "item": "ui.feature.automation2.workflows-tasks", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, { "roleLabel": "automation2-user", "description": { "en": "Automation2 User - Use automation2 flow builder", "de": "Automation2 Benutzer - Flow-Builder nutzen", - "fr": "Utilisateur Automation2 - Utiliser le flow builder" + "fr": "Utilisateur Automation2 - Utiliser le flow builder", }, "accessRules": [ {"context": "UI", "item": "ui.feature.automation2.editor", "view": True}, @@ -77,7 +90,20 @@ TEMPLATE_ROLES = [ {"context": "RESOURCE", "item": "resource.feature.automation2.node-types", "view": True}, {"context": "RESOURCE", "item": "resource.feature.automation2.execute", "view": True}, {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, - ] + ], + }, + { + "roleLabel": "automation2-admin", + "description": { + "en": "Automation2 Admin - Full UI and API for the instance; data remains user-scoped (MY)", + "de": "Automation2 Admin - Volle UI und API für die Instanz; Daten weiterhin benutzerspezifisch (MY)", + "fr": "Administrateur Automation2 - UI et API complets pour l'instance; donnees limitees a l'utilisateur (MY)", + }, + "accessRules": [ + {"context": "UI", "item": None, "view": True}, + {"context": "RESOURCE", "item": None, "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, + ], }, ] @@ -179,7 +205,7 @@ def getFeatureDefinition() -> Dict[str, Any]: "code": FEATURE_CODE, "label": FEATURE_LABEL, "icon": FEATURE_ICON, - "autoCreateInstance": True, + "autoCreateInstance": False, } diff --git a/modules/features/automation2/routeFeatureAutomation2.py b/modules/features/automation2/routeFeatureAutomation2.py index eaa49370..aa40f8bb 100644 --- a/modules/features/automation2/routeFeatureAutomation2.py +++ b/modules/features/automation2/routeFeatureAutomation2.py @@ -459,7 +459,7 @@ def get_workflows( active_run = None last_started_at = None for r in runs: - ts = r.get("_createdAt") + ts = r.get("sysCreatedAt") if ts and (last_started_at is None or ts > last_started_at): last_started_at = ts if r.get("status") in ("running", "paused"): @@ -475,7 +475,7 @@ def get_workflows( "runStatus": active_run.get("status") if active_run else None, "stuckAtNodeId": stuck_at_node_id, "stuckAtNodeLabel": stuck_at_node_label or stuck_at_node_id or "", - "createdAt": wf.get("_createdAt"), + "createdAt": wf.get("sysCreatedAt"), "lastStartedAt": last_started_at, }) return {"workflows": enriched} @@ -788,7 +788,7 @@ def get_tasks( context: RequestContext = Depends(getRequestContext), ) -> dict: """Get tasks - by default those assigned to current user, or all if no assignee filter. - Enriches each task with workflowLabel and createdAt (_createdAt). + Enriches each task with workflowLabel and createdAt (from sysCreatedAt). """ mandateId = _validateInstanceAccess(instanceId, context) a2 = getAutomation2Interface(context.user, mandateId, instanceId) @@ -801,7 +801,7 @@ def get_tasks( enriched.append({ **t, "workflowLabel": wf.get("label", t.get("workflowId", "")) if wf else t.get("workflowId", ""), - "createdAt": t.get("_createdAt"), + "createdAt": t.get("sysCreatedAt"), }) return {"tasks": enriched} diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 4a03bec9..151a96ce 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -20,6 +20,7 @@ from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel from modules.datamodels.datamodelChat import UserInputRequest +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp # ============================================================================= @@ -27,7 +28,7 @@ from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp # ============================================================================= -class ChatbotDocument(BaseModel): +class ChatbotDocument(PowerOnModel): """Documents attached to chatbot messages.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") messageId: str = Field(description="Foreign key to message") @@ -41,7 +42,7 @@ class ChatbotDocument(BaseModel): actionId: Optional[str] = Field(None, description="ID of the action that created this document") -class ChatbotMessage(BaseModel): +class ChatbotMessage(PowerOnModel): """Messages in chatbot conversations. Must match bridge format in memory.py.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") conversationId: str = Field(description="Foreign key to conversation") @@ -64,7 +65,7 @@ class ChatbotMessage(BaseModel): actionProgress: Optional[str] = Field(None, description="Action progress status") -class ChatbotLog(BaseModel): +class ChatbotLog(PowerOnModel): """Log entries for chatbot conversations.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") conversationId: str = Field(description="Foreign key to conversation") @@ -85,7 +86,7 @@ class ChatbotWorkflowModeEnum(str, Enum): WORKFLOW_CHATBOT = "Chatbot" -class ChatbotConversation(BaseModel): +class ChatbotConversation(PowerOnModel): """Chatbot conversation container. Per feature-instance isolation via featureInstanceId.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") featureInstanceId: str = Field(description="Feature instance ID for per-instance isolation") @@ -328,9 +329,8 @@ class ChatObjects: objectFields[fieldName] = value else: # Field not in model - treat as scalar if simple, otherwise filter out - # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector + # Underscore-prefixed keys (e.g. UI meta) pass through; sys* live on PowerOnModel subclasses if fieldName.startswith("_"): - # Metadata fields should be passed through to connector simpleFields[fieldName] = value elif isinstance(value, (str, int, float, bool, type(None))): simpleFields[fieldName] = value diff --git a/modules/features/chatbot/service.py b/modules/features/chatbot/service.py index 121ca29b..a98150b5 100644 --- a/modules/features/chatbot/service.py +++ b/modules/features/chatbot/service.py @@ -1222,23 +1222,21 @@ def _preflight_billing_check(services, mandateId: str, featureInstanceId: Option balanceCheck = billingService.checkBalance(0.01) if not balanceCheck.allowed: mid = str(getattr(services, "mandateId", None) or mandateId or "") - from modules.datamodels.datamodelBilling import BillingModelEnum from modules.serviceCenter.services.serviceBilling.billingExhaustedNotify import ( maybeEmailMandatePoolExhausted, ) - if balanceCheck.billingModel == BillingModelEnum.PREPAY_MANDATE: - u = getattr(services, "user", None) - ulabel = ( - (getattr(u, "email", None) or getattr(u, "username", None) or str(getattr(u, "id", ""))) - if u is not None else "" - ) - maybeEmailMandatePoolExhausted( - mid, - str(getattr(u, "id", "") if u is not None else ""), - ulabel, - float(balanceCheck.currentBalance or 0.0), - 0.01, - ) + u = getattr(services, "user", None) + ulabel = ( + (getattr(u, "email", None) or getattr(u, "username", None) or str(getattr(u, "id", ""))) + if u is not None else "" + ) + maybeEmailMandatePoolExhausted( + mid, + str(getattr(u, "id", "") if u is not None else ""), + ulabel, + float(balanceCheck.currentBalance or 0.0), + 0.01, + ) raise BillingService.InsufficientBalanceException.fromBalanceCheck( balanceCheck, mid, diff --git a/modules/features/commcoach/datamodelCommcoach.py b/modules/features/commcoach/datamodelCommcoach.py index 090640c6..635ba19a 100644 --- a/modules/features/commcoach/datamodelCommcoach.py +++ b/modules/features/commcoach/datamodelCommcoach.py @@ -7,6 +7,8 @@ Pydantic models for coaching contexts, sessions, messages, tasks, scores, and us from typing import Optional, List, Dict, Any from pydantic import BaseModel, Field from enum import Enum + +from modules.datamodels.datamodelBase import PowerOnModel import uuid @@ -73,7 +75,7 @@ class CoachingScoreTrend(str, Enum): # Database Models # ============================================================================ -class CoachingContext(BaseModel): +class CoachingContext(PowerOnModel): """A coaching context/dossier representing a topic the user is working on.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) userId: str = Field(description="Owner user ID (strict ownership)") @@ -91,11 +93,9 @@ class CoachingContext(BaseModel): lastSessionAt: Optional[str] = Field(default=None) rollingOverview: Optional[str] = Field(default=None, description="AI summary of older sessions for long context history") rollingOverviewUpToSessionCount: Optional[int] = Field(default=None, description="Session count covered by rollingOverview") - createdAt: Optional[str] = Field(default=None) - updatedAt: Optional[str] = Field(default=None) -class CoachingSession(BaseModel): +class CoachingSession(PowerOnModel): """A single coaching conversation session within a context.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) contextId: str = Field(description="FK to CoachingContext") @@ -115,11 +115,9 @@ class CoachingSession(BaseModel): emailSent: bool = Field(default=False) startedAt: Optional[str] = Field(default=None) endedAt: Optional[str] = Field(default=None) - createdAt: Optional[str] = Field(default=None) - updatedAt: Optional[str] = Field(default=None) -class CoachingMessage(BaseModel): +class CoachingMessage(PowerOnModel): """A single message in a coaching session.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) sessionId: str = Field(description="FK to CoachingSession") @@ -130,10 +128,9 @@ class CoachingMessage(BaseModel): contentType: CoachingMessageContentType = Field(default=CoachingMessageContentType.TEXT) audioRef: Optional[str] = Field(default=None, description="Reference to audio file") metadata: Optional[str] = Field(default=None, description="JSON: token count, voice info, etc.") - createdAt: Optional[str] = Field(default=None) -class CoachingTask(BaseModel): +class CoachingTask(PowerOnModel): """A task/checklist item assigned within a coaching context.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) contextId: str = Field(description="FK to CoachingContext") @@ -146,11 +143,9 @@ class CoachingTask(BaseModel): priority: CoachingTaskPriority = Field(default=CoachingTaskPriority.MEDIUM) dueDate: Optional[str] = Field(default=None) completedAt: Optional[str] = Field(default=None) - createdAt: Optional[str] = Field(default=None) - updatedAt: Optional[str] = Field(default=None) -class CoachingScore(BaseModel): +class CoachingScore(PowerOnModel): """A competence score for a dimension, recorded after a session.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) contextId: str = Field(description="FK to CoachingContext") @@ -161,17 +156,14 @@ class CoachingScore(BaseModel): score: float = Field(ge=0.0, le=100.0) trend: CoachingScoreTrend = Field(default=CoachingScoreTrend.STABLE) evidence: Optional[str] = Field(default=None, description="AI reasoning for the score") - createdAt: Optional[str] = Field(default=None) -class CoachingUserProfile(BaseModel): +class CoachingUserProfile(PowerOnModel): """Per-user coaching profile and preferences.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) userId: str = Field(description="Owner user ID") mandateId: str = Field(description="Mandate ID") instanceId: str = Field(description="Feature instance ID") - preferredLanguage: str = Field(default="de-DE") - preferredVoice: Optional[str] = Field(default=None, description="Google TTS voice name") dailyReminderTime: Optional[str] = Field(default=None, description="HH:MM format") dailyReminderEnabled: bool = Field(default=False) emailSummaryEnabled: bool = Field(default=True) @@ -180,15 +172,13 @@ class CoachingUserProfile(BaseModel): totalSessions: int = Field(default=0) totalMinutes: int = Field(default=0) lastSessionAt: Optional[str] = Field(default=None) - createdAt: Optional[str] = Field(default=None) - updatedAt: Optional[str] = Field(default=None) # ============================================================================ # Iteration 2: Personas # ============================================================================ -class CoachingPersona(BaseModel): +class CoachingPersona(PowerOnModel): """A roleplay persona for coaching sessions.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) userId: str = Field(description="Owner user ID ('system' for builtins)") @@ -201,35 +191,13 @@ class CoachingPersona(BaseModel): gender: Optional[str] = Field(default=None, description="m or f") category: str = Field(default="builtin", description="'builtin' or 'custom'") isActive: bool = Field(default=True) - createdAt: Optional[str] = Field(default=None) - updatedAt: Optional[str] = Field(default=None) - - -# ============================================================================ -# Iteration 2: Documents -# ============================================================================ - -class CoachingDocument(BaseModel): - """A document attached to a coaching context.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4())) - contextId: str = Field(description="FK to CoachingContext") - userId: str = Field(description="Owner user ID") - mandateId: str = Field(description="Mandate ID") - instanceId: Optional[str] = Field(default=None) - fileName: str = Field(description="Original file name") - mimeType: str = Field(default="application/octet-stream") - fileSize: int = Field(default=0) - extractedText: Optional[str] = Field(default=None, description="Text content extracted from file") - summary: Optional[str] = Field(default=None, description="AI-generated summary") - fileRef: Optional[str] = Field(default=None, description="Reference to file in storage") - createdAt: Optional[str] = Field(default=None) # ============================================================================ # Iteration 2: Badges / Gamification # ============================================================================ -class CoachingBadge(BaseModel): +class CoachingBadge(PowerOnModel): """An achievement badge awarded to a user.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) userId: str = Field(description="Owner user ID") @@ -237,7 +205,6 @@ class CoachingBadge(BaseModel): instanceId: str = Field(description="Feature instance ID") badgeKey: str = Field(description="Badge identifier, e.g. 'streak_7'") awardedAt: Optional[str] = Field(default=None) - createdAt: Optional[str] = Field(default=None) # ============================================================================ @@ -282,8 +249,6 @@ class UpdateTaskStatusRequest(BaseModel): class UpdateProfileRequest(BaseModel): - preferredLanguage: Optional[str] = None - preferredVoice: Optional[str] = None dailyReminderTime: Optional[str] = None dailyReminderEnabled: Optional[bool] = None emailSummaryEnabled: Optional[bool] = None diff --git a/modules/features/commcoach/interfaceFeatureCommcoach.py b/modules/features/commcoach/interfaceFeatureCommcoach.py index e612c6ba..825fca5d 100644 --- a/modules/features/commcoach/interfaceFeatureCommcoach.py +++ b/modules/features/commcoach/interfaceFeatureCommcoach.py @@ -269,34 +269,6 @@ class CommcoachObjects: from .datamodelCommcoach import CoachingPersona return self.db.recordDelete(CoachingPersona, personaId) - # ========================================================================= - # Documents - # ========================================================================= - - def getDocuments(self, contextId: str, userId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - records = self.db.getRecordset(CoachingDocument, recordFilter={"contextId": contextId, "userId": userId}) - records.sort(key=lambda r: r.get("createdAt") or "", reverse=True) - return records - - def getDocument(self, documentId: str) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - records = self.db.getRecordset(CoachingDocument, recordFilter={"id": documentId}) - return records[0] if records else None - - def createDocument(self, data: Dict[str, Any]) -> Dict[str, Any]: - from .datamodelCommcoach import CoachingDocument - data["createdAt"] = getIsoTimestamp() - return self.db.recordCreate(CoachingDocument, data) - - def updateDocument(self, documentId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - return self.db.recordModify(CoachingDocument, documentId, updates) - - def deleteDocument(self, documentId: str) -> bool: - from .datamodelCommcoach import CoachingDocument - return self.db.recordDelete(CoachingDocument, documentId) - # ========================================================================= # Badges # ========================================================================= diff --git a/modules/features/commcoach/mainCommcoach.py b/modules/features/commcoach/mainCommcoach.py index 69ac6b1c..9d949e13 100644 --- a/modules/features/commcoach/mainCommcoach.py +++ b/modules/features/commcoach/mainCommcoach.py @@ -61,18 +61,13 @@ DATA_OBJECTS = [ { "objectKey": "data.feature.commcoach.CoachingUserProfile", "label": {"en": "User Profile", "de": "Benutzerprofil", "fr": "Profil utilisateur"}, - "meta": {"table": "CoachingUserProfile", "fields": ["id", "userId", "preferredLanguage"]} + "meta": {"table": "CoachingUserProfile", "fields": ["id", "userId", "dailyReminderEnabled"]} }, { "objectKey": "data.feature.commcoach.CoachingPersona", "label": {"en": "Coaching Persona", "de": "Coaching-Persona", "fr": "Persona coaching"}, "meta": {"table": "CoachingPersona", "fields": ["id", "key", "label", "gender"]} }, - { - "objectKey": "data.feature.commcoach.CoachingDocument", - "label": {"en": "Coaching Document", "de": "Coaching-Dokument", "fr": "Document coaching"}, - "meta": {"table": "CoachingDocument", "fields": ["id", "contextId", "fileName"]} - }, { "objectKey": "data.feature.commcoach.CoachingBadge", "label": {"en": "Coaching Badge", "de": "Coaching-Auszeichnung", "fr": "Badge coaching"}, @@ -114,12 +109,27 @@ RESOURCE_OBJECTS = [ ] TEMPLATE_ROLES = [ + { + "roleLabel": "commcoach-viewer", + "description": { + "en": "Communication Coach Viewer - View coaching data (read-only)", + "de": "Kommunikations-Coach Betrachter - Coaching-Daten ansehen (nur lesen)", + "fr": "Visualiseur Coach Communication - Consulter les donnees coaching (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.commcoach.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.commcoach.coaching", "view": True}, + {"context": "UI", "item": "ui.feature.commcoach.dossier", "view": True}, + {"context": "UI", "item": "ui.feature.commcoach.settings", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, { "roleLabel": "commcoach-user", "description": { "en": "Communication Coach User - Can manage own coaching contexts and sessions", "de": "Kommunikations-Coach Benutzer - Kann eigene Coaching-Kontexte und Sessions verwalten", - "fr": "Utilisateur Coach Communication - Peut gerer ses propres contextes et sessions" + "fr": "Utilisateur Coach Communication - Peut gerer ses propres contextes et sessions", }, "accessRules": [ {"context": "UI", "item": "ui.feature.commcoach.dashboard", "view": True}, @@ -137,7 +147,20 @@ TEMPLATE_ROLES = [ {"context": "RESOURCE", "item": "resource.feature.commcoach.session.start", "view": True}, {"context": "RESOURCE", "item": "resource.feature.commcoach.session.complete", "view": True}, {"context": "RESOURCE", "item": "resource.feature.commcoach.task.manage", "view": True}, - ] + ], + }, + { + "roleLabel": "commcoach-admin", + "description": { + "en": "Communication Coach Admin - All UI and API actions; data scoped to own records", + "de": "Kommunikations-Coach Admin - Alle UI- und API-Aktionen; Daten nur eigene Datensaetze", + "fr": "Administrateur Coach Communication - Toute l'UI et les API; donnees propres", + }, + "accessRules": [ + {"context": "UI", "item": None, "view": True}, + {"context": "RESOURCE", "item": None, "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, + ], }, ] @@ -147,7 +170,7 @@ def getFeatureDefinition() -> Dict[str, Any]: "code": FEATURE_CODE, "label": FEATURE_LABEL, "icon": FEATURE_ICON, - "autoCreateInstance": True, + "autoCreateInstance": False, } diff --git a/modules/features/commcoach/routeFeatureCommcoach.py b/modules/features/commcoach/routeFeatureCommcoach.py index 9074d2ba..ccb4d342 100644 --- a/modules/features/commcoach/routeFeatureCommcoach.py +++ b/modules/features/commcoach/routeFeatureCommcoach.py @@ -2,7 +2,7 @@ # All rights reserved. """ CommCoach routes for the backend API. -Implements coaching context management, session streaming, tasks, dashboard, and voice endpoints. +Implements coaching context management, session streaming, tasks, and dashboard. """ import logging @@ -26,7 +26,7 @@ from .datamodelCommcoach import ( CoachingContext, CoachingContextStatus, CoachingSession, CoachingSessionStatus, CoachingMessage, CoachingMessageRole, CoachingMessageContentType, CoachingTask, CoachingTaskStatus, - CoachingPersona, CoachingDocument, CoachingBadge, + CoachingPersona, CoachingBadge, CreateContextRequest, UpdateContextRequest, SendMessageRequest, CreateTaskRequest, UpdateTaskRequest, UpdateTaskStatusRequest, UpdateProfileRequest, @@ -334,9 +334,8 @@ async def startSession( try: from modules.interfaces.interfaceVoiceObjects import getVoiceInterface voiceInterface = getVoiceInterface(context.user, mandateId) - profile = interface.getProfile(userId, instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" - voiceName = profile.get("preferredVoice") if profile else None + from .serviceCommcoach import _getUserVoicePrefs + language, voiceName = _getUserVoicePrefs(userId, mandateId) from .serviceCommcoach import _stripMarkdownForTts ttsResult = await voiceInterface.textToSpeech( text=_stripMarkdownForTts(greetingText), @@ -574,8 +573,8 @@ async def sendAudioStream( if not audioBody: raise HTTPException(status_code=400, detail="No audio data received") - profile = interface.getProfile(str(context.user.id), instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" + from .serviceCommcoach import _getUserVoicePrefs + language, _ = _getUserVoicePrefs(str(context.user.id), mandateId) contextId = session.get("contextId") service = CommcoachService(context.user, mandateId, instanceId) @@ -839,73 +838,6 @@ async def updateProfile( return {"profile": updated} -# ========================================================================= -# Voice Endpoints -# ========================================================================= - -@router.get("/{instanceId}/voice/languages") -@limiter.limit("30/minute") -async def getVoiceLanguages( - request: Request, - instanceId: str, - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - languagesResult = await voiceInterface.getAvailableLanguages() - languageList = languagesResult.get("languages", []) if isinstance(languagesResult, dict) else languagesResult - return {"languages": languageList} - - -@router.get("/{instanceId}/voice/voices") -@limiter.limit("30/minute") -async def getVoiceVoices( - request: Request, - instanceId: str, - language: str = "de-DE", - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - voicesResult = await voiceInterface.getAvailableVoices(language) - voiceList = voicesResult.get("voices", []) if isinstance(voicesResult, dict) else voicesResult - return {"voices": voiceList} - - -@router.post("/{instanceId}/voice/tts") -@limiter.limit("10/minute") -async def testVoice( - request: Request, - instanceId: str, - context: RequestContext = Depends(getRequestContext), -): - """TTS preview / voice test.""" - mandateId = _validateInstanceAccess(instanceId, context) - body = await request.json() - text = body.get("text", "Hallo, ich bin dein Coaching-Assistent.") - language = body.get("language", "de-DE") - voiceId = body.get("voiceId") - - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - - try: - result = await voiceInterface.textToSpeech(text=text, languageCode=language, voiceName=voiceId) - if result and isinstance(result, dict): - audioContent = result.get("audioContent") - if audioContent: - audioB64 = base64.b64encode( - audioContent if isinstance(audioContent, bytes) else audioContent.encode() - ).decode() - return {"success": True, "audio": audioB64, "format": "mp3", "text": text} - return {"success": False, "error": "TTS returned no audio"} - except Exception as e: - logger.error(f"Voice test failed: {e}") - raise HTTPException(status_code=500, detail=f"TTS test failed: {str(e)}") - - # ========================================================================= # Export Endpoints (Iteration 2) # ========================================================================= @@ -1074,202 +1006,6 @@ async def deletePersonaRoute( return {"deleted": True} -# ========================================================================= -# Document Endpoints (Iteration 2) -# ========================================================================= - -@router.get("/{instanceId}/contexts/{contextId}/documents") -@limiter.limit("60/minute") -async def listDocuments( - request: Request, - instanceId: str, - contextId: str, - context: RequestContext = Depends(getRequestContext), -): - _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - userId = str(context.user.id) - docs = interface.getDocuments(contextId, userId) - return {"documents": docs} - - -@router.post("/{instanceId}/contexts/{contextId}/documents") -@limiter.limit("10/minute") -async def uploadDocument( - request: Request, - instanceId: str, - contextId: str, - context: RequestContext = Depends(getRequestContext), -): - """Upload a document and bind it to a context. Stores file in Management DB.""" - mandateId = _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - userId = str(context.user.id) - - ctx = interface.getContext(contextId) - if not ctx: - raise HTTPException(status_code=404, detail="Context not found") - _validateOwnership(ctx, context) - - form = await request.form() - file = form.get("file") - if not file or not hasattr(file, "read"): - raise HTTPException(status_code=400, detail="No file uploaded") - - content = await file.read() - fileName = getattr(file, "filename", "document") - mimeType = getattr(file, "content_type", "application/octet-stream") - fileSize = len(content) - - if not content: - raise HTTPException(status_code=400, detail="Leere Datei hochgeladen") - - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface(currentUser=context.user) - fileItem, _dupType = mgmtInterface.saveUploadedFile(content, fileName) - fileRef = fileItem.id - - extractedText = _extractText(content, mimeType, fileName) - summary = None - if extractedText and len(extractedText.strip()) > 50: - try: - from .serviceCommcoach import CommcoachService - service = CommcoachService(context.user, mandateId, instanceId) - aiResp = await service._callAi( - "Du fasst Dokumente in 2-3 Saetzen zusammen.", - f"Fasse folgendes Dokument zusammen:\n\n{extractedText[:3000]}" - ) - if aiResp and aiResp.errorCount == 0 and aiResp.content: - summary = aiResp.content.strip() - except Exception as e: - logger.warning(f"Document summary failed: {e}") - - docData = CoachingDocument( - contextId=contextId, - userId=userId, - mandateId=mandateId, - instanceId=instanceId, - fileName=fileName, - mimeType=mimeType, - fileSize=fileSize, - extractedText=extractedText[:10000] if extractedText else None, - summary=summary, - fileRef=fileRef, - ).model_dump() - created = interface.createDocument(docData) - return {"document": created} - - -@router.delete("/{instanceId}/documents/{documentId}") -@limiter.limit("10/minute") -async def deleteDocumentRoute( - request: Request, - instanceId: str, - documentId: str, - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - - doc = interface.getDocument(documentId) - if not doc: - raise HTTPException(status_code=404, detail="Document not found") - _validateOwnership(doc, context) - - fileRef = doc.get("fileRef") - if fileRef: - try: - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface( - currentUser=context.user, mandateId=mandateId, featureInstanceId=instanceId - ) - mgmtInterface.deleteFile(fileRef) - except Exception as e: - logger.warning(f"Failed to delete file {fileRef}: {e}") - - interface.deleteDocument(documentId) - return {"deleted": True} - - -def _extractText(content: bytes, mimeType: str, fileName: str) -> Optional[str]: - """Extract text from uploaded file content (TXT, MD, HTML, PDF, DOCX, XLSX, PPTX).""" - import io - - lowerName = fileName.lower() - try: - if mimeType in ("text/plain",) or lowerName.endswith(".txt"): - return content.decode("utf-8", errors="replace") - - if mimeType in ("text/markdown",) or lowerName.endswith(".md"): - return content.decode("utf-8", errors="replace") - - if mimeType in ("text/html",) or lowerName.endswith((".html", ".htm")): - from html.parser import HTMLParser - class _Strip(HTMLParser): - def __init__(self): - super().__init__() - self._parts: list[str] = [] - def handle_data(self, d): - self._parts.append(d) - def result(self): - return " ".join(self._parts) - parser = _Strip() - parser.feed(content.decode("utf-8", errors="replace")) - return parser.result() - - if "pdf" in mimeType or lowerName.endswith(".pdf"): - try: - from PyPDF2 import PdfReader - reader = PdfReader(io.BytesIO(content)) - return "".join(page.extract_text() or "" for page in reader.pages) - except ImportError: - logger.warning("PyPDF2 not installed, cannot extract PDF text") - return None - - if "wordprocessingml" in mimeType or lowerName.endswith(".docx"): - try: - from docx import Document - doc = Document(io.BytesIO(content)) - return "\n".join(p.text for p in doc.paragraphs if p.text) - except ImportError: - logger.warning("python-docx not installed, cannot extract DOCX text") - return None - - if "spreadsheetml" in mimeType or lowerName.endswith(".xlsx"): - try: - from openpyxl import load_workbook - wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True) - parts: list[str] = [] - for ws in wb.worksheets: - for row in ws.iter_rows(values_only=True): - cells = [str(c) for c in row if c is not None] - if cells: - parts.append("\t".join(cells)) - return "\n".join(parts) - except ImportError: - logger.warning("openpyxl not installed, cannot extract XLSX text") - return None - - if "presentationml" in mimeType or lowerName.endswith(".pptx"): - try: - from pptx import Presentation - prs = Presentation(io.BytesIO(content)) - parts = [] - for slide in prs.slides: - for shape in slide.shapes: - if shape.has_text_frame: - parts.append(shape.text_frame.text) - return "\n".join(parts) - except ImportError: - logger.warning("python-pptx not installed, cannot extract PPTX text") - return None - - logger.info(f"No text extractor for {fileName} (mime={mimeType})") - except Exception as e: - logger.warning(f"Text extraction failed for {fileName}: {e}") - return None - - # ========================================================================= # Badge + Score History Endpoints (Iteration 2) # ========================================================================= diff --git a/modules/features/commcoach/serviceCommcoach.py b/modules/features/commcoach/serviceCommcoach.py index bf5ec281..5e5aa810 100644 --- a/modules/features/commcoach/serviceCommcoach.py +++ b/modules/features/commcoach/serviceCommcoach.py @@ -33,6 +33,7 @@ from .serviceCommcoachContextRetrieval import ( buildSessionSummariesForPrompt, findSessionByDate, searchSessionsByTopic, + searchSessionsByTopicRag, _parseDateFromMessage, PREVIOUS_SESSION_SUMMARIES_COUNT, ROLLING_OVERVIEW_SESSION_THRESHOLD, @@ -42,6 +43,30 @@ from .serviceCommcoachContextRetrieval import ( logger = logging.getLogger(__name__) +def _getUserVoicePrefs(userId: str, mandateId: Optional[str] = None) -> tuple: + """Load voice language and voiceName from central UserVoicePreferences. + Returns (language, voiceName) tuple.""" + try: + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.security.rootAccess import getRootInterface + rootIf = getRootInterface() + prefs = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId, "mandateId": mandateId} + ) + if not prefs and mandateId: + prefs = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId} + ) + if prefs: + p = prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump() + return (p.get("ttsLanguage") or p.get("sttLanguage") or "de-DE", p.get("ttsVoice")) + except Exception as e: + logger.warning(f"Failed to load UserVoicePreferences for user={userId}: {e}") + return ("de-DE", None) + + def _stripMarkdownForTts(text: str) -> str: """Strip markdown formatting so TTS reads clean speech text.""" t = text @@ -159,9 +184,7 @@ async def _generateAndEmitTts(sessionId: str, speechText: str, currentUser, mand from modules.interfaces.interfaceVoiceObjects import getVoiceInterface import base64 voiceInterface = getVoiceInterface(currentUser, mandateId) - profile = interface.getProfile(str(currentUser.id), instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" - voiceName = profile.get("preferredVoice") if profile else None + language, voiceName = _getUserVoicePrefs(str(currentUser.id), mandateId) ttsResult = await voiceInterface.textToSpeech( text=_stripMarkdownForTts(speechText), languageCode=language, @@ -196,60 +219,36 @@ def _resolveFileNameAndMime(title: str) -> tuple: async def _saveOrUpdateDocument(doc: Dict[str, Any], contextId: str, userId: str, mandateId: str, instanceId: str, interface, sessionId: str, user=None): - """Save a new document or update an existing one. Stores file in Management DB.""" - from .datamodelCommcoach import CoachingDocument + """Save a document as platform FileItem (no CoachingDocument).""" try: - docId = doc.get("id") title = doc.get("title", "Dokument") content = doc.get("content", "") contentBytes = content.encode("utf-8") fileName, mimeType = _resolveFileNameAndMime(title) - fileRef = None - try: - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface( - currentUser=user, mandateId=mandateId, featureInstanceId=instanceId - ) - fileItem = mgmtInterface.createFile(name=fileName, mimeType=mimeType, content=contentBytes) - mgmtInterface.createFileData(fileItem.id, contentBytes) - fileRef = fileItem.id - except Exception as e: - logger.warning(f"Failed to store document in file DB: {e}") + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + mgmtInterface = interfaceDbManagement.getInterface( + currentUser=user, mandateId=mandateId, featureInstanceId=instanceId + ) + fileItem = mgmtInterface.createFile(name=fileName, mimeType=mimeType, content=contentBytes) + mgmtInterface.createFileData(fileItem.id, contentBytes) + + from modules.datamodels.datamodelFiles import FileItem as FileItemModel + mgmtInterface.db.recordModify(FileItemModel, fileItem.id, { + "scope": "featureInstance", + "featureInstanceId": instanceId, + "mandateId": mandateId, + }) + + await emitSessionEvent(sessionId, "documentCreated", { + "id": fileItem.id, "fileName": fileName, "fileSize": len(contentBytes), + }) + logger.info(f"Document saved as platform FileItem: {fileItem.id} ({title})") - if docId: - updates = { - "fileName": fileName, - "mimeType": mimeType, - "extractedText": content, - "summary": title, - "fileSize": len(contentBytes), - } - if fileRef: - updates["fileRef"] = fileRef - updated = interface.updateDocument(docId, updates) - if updated: - await emitSessionEvent(sessionId, "documentUpdated", updated) - logger.info(f"Document updated: {docId} ({title})") - else: - logger.warning(f"Document update failed, id not found: {docId}") - else: - docData = CoachingDocument( - contextId=contextId, - userId=userId, - mandateId=mandateId, - instanceId=instanceId, - fileName=fileName, - mimeType=mimeType, - fileSize=len(contentBytes), - extractedText=content, - summary=title, - fileRef=fileRef, - ).model_dump() - created = interface.createDocument(docData) - await emitSessionEvent(sessionId, "documentCreated", created) except Exception as e: - logger.warning(f"Failed to save/update document: {e}") + logger.warning(f"Failed to save document as FileItem: {e}") + + async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, Any]], callAiFn) -> Dict[str, Any]: @@ -269,17 +268,60 @@ async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, A return {"read": [], "update": [], "create": [], "noDocumentAction": True} -def _loadDocumentContents(docIds: List[str], interface) -> List[Dict[str, Any]]: - """Load full extractedText for the given document IDs.""" - results = [] - for docId in docIds[:DOC_INTENT_MAX_DOCS]: - doc = interface.getDocument(docId) - if doc and doc.get("extractedText"): - results.append({ - "id": doc.get("id", ""), - "title": doc.get("summary") or doc.get("fileName", ""), - "content": doc.get("extractedText", "")[:DOC_CONTENT_MAX_CHARS], +def _getPlatformFileList(mandateId: str = None, instanceId: str = None) -> List[Dict[str, Any]]: + """Get list of platform FileItems for this feature instance (for doc intent detection).""" + try: + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + records = mgmtIf.db.getRecordset( + FileItem, recordFilter={"featureInstanceId": instanceId} + ) if instanceId else [] + result = [] + for r in records: + d = r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else {} + result.append({ + "id": d.get("id", ""), + "fileName": d.get("fileName") or d.get("name") or "Dokument", + "summary": d.get("fileName") or "", }) + return result + except Exception as e: + logger.warning(f"Failed to load platform file list: {e}") + return [] + + +def _loadDocumentContents(docIds: List[str], interface, mandateId: str = None, instanceId: str = None) -> List[Dict[str, Any]]: + """Load file content for given IDs from platform FileItem store.""" + results = [] + try: + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + for fId in docIds[:DOC_INTENT_MAX_DOCS]: + fileRecords = mgmtIf.db.getRecordset(FileItem, recordFilter={"id": fId}) + if fileRecords: + f = fileRecords[0] if isinstance(fileRecords[0], dict) else fileRecords[0].model_dump() + content = "" + try: + from modules.datamodels.datamodelKnowledge import FileContentIndex + idxRecords = mgmtIf.db.getRecordset(FileContentIndex, recordFilter={"fileId": fId}) + if idxRecords: + idx = idxRecords[0] if isinstance(idxRecords[0], dict) else idxRecords[0].model_dump() + content = (idx.get("extractedText") or "")[:DOC_CONTENT_MAX_CHARS] + except Exception: + pass + results.append({ + "id": fId, + "title": f.get("fileName") or f.get("name") or "Dokument", + "content": content, + }) + except Exception as e: + logger.warning(f"Failed to load document contents from platform: {e}") return results @@ -319,20 +361,42 @@ def _resolvePersona(session: Optional[Dict[str, Any]], interface) -> Optional[Di return None -def _getDocumentSummaries(contextId: str, userId: str, interface) -> Optional[List[str]]: - """Get document summaries for context to include in the AI prompt.""" +def _getDocumentSummaries(contextId: str, userId: str, interface, + mandateId: str = None, instanceId: str = None) -> Optional[List[str]]: + """Get document summaries from platform FileItems (UDL) for the coaching instance.""" try: - docs = interface.getDocuments(contextId, userId) + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + files = mgmtIf.db.getRecordset( + FileItem, recordFilter={"featureInstanceId": instanceId} + ) if instanceId else [] summaries = [] - for doc in docs[:5]: - summary = doc.get("summary") - if summary: - summaries.append(f"[{doc.get('fileName', 'Dokument')}] {summary}") - elif doc.get("extractedText"): - summaries.append(f"[{doc.get('fileName', 'Dokument')}] {doc['extractedText'][:200]}...") + for f in files[:10]: + fData = f if isinstance(f, dict) else f.model_dump() if hasattr(f, "model_dump") else {} + name = fData.get("fileName") or fData.get("name") or "Dokument" + fId = fData.get("id") + snippet = None + if fId: + try: + from modules.datamodels.datamodelKnowledge import FileContentIndex + idxRecords = mgmtIf.db.getRecordset( + FileContentIndex, recordFilter={"fileId": fId} + ) + if idxRecords: + idx = idxRecords[0] if isinstance(idxRecords[0], dict) else idxRecords[0].model_dump() + snippet = (idx.get("extractedText") or "")[:200] + except Exception: + pass + if snippet: + summaries.append(f"[{name}] {snippet}...") + else: + summaries.append(f"[{name}]") return summaries if summaries else None except Exception as e: - logger.warning(f"Failed to load document summaries for context {contextId}: {e}") + logger.warning(f"Failed to load platform file summaries for instance {instanceId}: {e}") return None @@ -427,18 +491,22 @@ class CommcoachService: ) persona = _resolvePersona(session, interface) - documentSummaries = _getDocumentSummaries(contextId, self.userId, interface) + documentSummaries = _getDocumentSummaries( + contextId, self.userId, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) # Document intent detection (pre-AI-call) referencedDocumentContents = None - allDocs = interface.getDocuments(contextId, self.userId) if documentSummaries else [] + allDocs = _getPlatformFileList(self.mandateId, self.instanceId) if documentSummaries else [] if allDocs: await emitSessionEvent(sessionId, "status", {"label": "Dokumente werden geprueft..."}) docIntent = await _resolveDocumentIntent(combinedUserPrompt, allDocs, self._callAi) if not docIntent.get("noDocumentAction"): docIdsToLoad = list(set((docIntent.get("read") or []) + (docIntent.get("update") or []))) if docIdsToLoad: - referencedDocumentContents = _loadDocumentContents(docIdsToLoad, interface) + referencedDocumentContents = _loadDocumentContents( + docIdsToLoad, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) systemPrompt = aiPrompts.buildCoachingSystemPrompt( context, @@ -536,7 +604,9 @@ class CommcoachService: session = interface.getSession(sessionId) persona = _resolvePersona(session, interface) - documentSummaries = _getDocumentSummaries(contextId, self.userId, interface) + documentSummaries = _getDocumentSummaries( + contextId, self.userId, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) systemPrompt = aiPrompts.buildCoachingSystemPrompt( context, previousMessages, tasks, @@ -966,10 +1036,29 @@ class CommcoachService: result["rollingOverview"] = rollingOverview elif intent == RetrievalIntent.RECALL_TOPIC: - retrieved = searchSessionsByTopic(completedSessions, userContent) + retrieved = list(searchSessionsByTopic(completedSessions, userContent)) + queryVector = await self._embedUserQuery(userContent) + if queryVector: + ragHits = searchSessionsByTopicRag( + userContent, + self.userId, + self.instanceId, + mandateId=self.mandateId, + queryVector=queryVector, + ) + for hit in ragHits: + content = (hit.get("content") or "").strip() + if not content: + continue + retrieved.append({ + "summary": content[:450], + "date": "", + "source": "rag", + "ragSourceLabel": hit.get("fileName") or "Mandantenwissen", + }) result["retrievedByTopic"] = retrieved if retrieved: - logger.info(f"Topic recall: found {len(retrieved)} sessions for query") + logger.info(f"Topic recall: {len(retrieved)} item(s) (sessions + optional RAG)") result["previousSessionSummaries"] = buildSessionSummariesForPrompt( allSessions, excludeSessionId=sessionId, limit=PREVIOUS_SESSION_SUMMARIES_COUNT ) @@ -1032,3 +1121,31 @@ class CommcoachService: ) ) return await aiService.callAi(aiRequest) + + async def _embedUserQuery(self, text: str) -> Optional[List[float]]: + """Embedding for mandate-wide RAG (same ServiceCenter AI service as coaching calls).""" + snippet = (text or "").strip()[:2000] + if not snippet: + return None + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + + serviceContext = ServiceCenterContext( + user=self.currentUser, + mandate_id=self.mandateId, + feature_instance_id=self.instanceId, + ) + aiService = getService("ai", serviceContext) + await aiService.ensureAiObjectsInitialized() + try: + response = await aiService.callEmbedding([snippet]) + except Exception as e: + logger.warning(f"CommCoach RAG embedding failed: {e}") + return None + if not response or response.errorCount > 0: + return None + embs = (response.metadata or {}).get("embeddings") or [] + vec = embs[0] if embs else None + if isinstance(vec, list) and len(vec) > 0: + return vec + return None diff --git a/modules/features/commcoach/serviceCommcoachAi.py b/modules/features/commcoach/serviceCommcoachAi.py index 7ba52f58..97deb373 100644 --- a/modules/features/commcoach/serviceCommcoachAi.py +++ b/modules/features/commcoach/serviceCommcoachAi.py @@ -229,12 +229,18 @@ WICHTIG: Antworte NUR mit dem JSON-Objekt. Kein Text vor oder nach dem JSON.""" prompt += f"\n{retrievedSession.get('summary', '')[:500]}" if retrievedByTopic: - prompt += "\n\nRelevante Sessions zum angefragten Thema:" - for s in retrievedByTopic[:3]: - summary = s.get("summary", "") + prompt += "\n\nRelevante Sessions und Mandantenwissen zum angefragten Thema:" + for s in retrievedByTopic[:5]: + summary = s.get("summary", s.get("content", "")) + if not summary: + continue dateStr = s.get("date", "") - if summary: - prompt += f"\n- [{dateStr}] {summary[:300]}" + if s.get("source") == "rag": + label = s.get("ragSourceLabel") or "Mandantenwissen" + prompt += f"\n- [Wissen: {label}] {summary[:320]}" + else: + prefix = f"[{dateStr}] " if dateStr else "" + prompt += f"\n- {prefix}{summary[:300]}" if openTasks: prompt += "\n\nOffene Aufgaben:" diff --git a/modules/features/commcoach/serviceCommcoachContextRetrieval.py b/modules/features/commcoach/serviceCommcoachContextRetrieval.py index d673b04a..f1ccb9a3 100644 --- a/modules/features/commcoach/serviceCommcoachContextRetrieval.py +++ b/modules/features/commcoach/serviceCommcoachContextRetrieval.py @@ -172,20 +172,48 @@ def searchSessionsByTopic( def searchSessionsByTopicRag( - sessions: List[Dict[str, Any]], query: str, - maxResults: int = TOPIC_SEARCH_MAX_RESULTS, - embeddingProvider: Optional[Any] = None, + userId: str, + instanceId: str, + mandateId: str = None, + queryVector: List[float] = None, ) -> List[Dict[str, Any]]: + """Search using platform RAG (semantic search across mandate-wide knowledge data). + + Requires a pre-computed queryVector (embedding). The caller is responsible + for generating the embedding via AiService.callEmbedding before invoking this. """ - Phase 7 RAG: Semantic search via embeddings. - When embeddingProvider is None, falls back to keyword search. - Future: Pass embeddingProvider that has embed(text) -> vector and similarity search. - """ - if embeddingProvider is None: - return searchSessionsByTopic(sessions, query, maxResults) - # TODO: When embedding API exists: embed query, embed session summaries, cosine similarity - return searchSessionsByTopic(sessions, query, maxResults) + if not queryVector: + logger.warning("searchSessionsByTopicRag called without queryVector, skipping RAG search") + return [] + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as _getKnowledgeInterface + + knowledgeDb = _getKnowledgeInterface() + + results = knowledgeDb.semanticSearch( + queryVector=queryVector, + userId=userId, + featureInstanceId=instanceId, + mandateId=mandateId, + isSysAdmin=False, + limit=TOPIC_SEARCH_MAX_RESULTS, + ) + + formatted = [] + for r in (results or []): + rData = r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else {} + contextRef = rData.get("contextRef") or {} + formatted.append({ + "source": "rag", + "content": rData.get("data") or rData.get("summary") or "", + "fileName": contextRef.get("containerPath") or "RAG-Ergebnis", + "score": rData.get("_score") or 0, + }) + return formatted + except Exception as e: + logger.warning(f"RAG search failed for query '{query[:50]}': {e}") + return [] def buildSessionSummariesForPrompt( diff --git a/modules/features/commcoach/tests/test_datamodel.py b/modules/features/commcoach/tests/test_datamodel.py index fb39ba34..05d174c5 100644 --- a/modules/features/commcoach/tests/test_datamodel.py +++ b/modules/features/commcoach/tests/test_datamodel.py @@ -136,7 +136,6 @@ class TestCoachingUserProfile: profile = CoachingUserProfile( userId="u1", mandateId="m1", instanceId="i1", ) - assert profile.preferredLanguage == "de-DE" assert profile.dailyReminderEnabled is False assert profile.emailSummaryEnabled is True assert profile.streakDays == 0 diff --git a/modules/features/commcoach/tests/test_mainCommcoach.py b/modules/features/commcoach/tests/test_mainCommcoach.py index 85d85cf6..6be563b6 100644 --- a/modules/features/commcoach/tests/test_mainCommcoach.py +++ b/modules/features/commcoach/tests/test_mainCommcoach.py @@ -31,7 +31,7 @@ class TestFeatureDefinition: assert defn["code"] == "commcoach" assert "label" in defn assert "icon" in defn - assert defn["autoCreateInstance"] is True + assert defn["autoCreateInstance"] is False class TestRbacObjects: diff --git a/modules/features/neutralization/datamodelFeatureNeutralizer.py b/modules/features/neutralization/datamodelFeatureNeutralizer.py index e7b46c4d..cc111950 100644 --- a/modules/features/neutralization/datamodelFeatureNeutralizer.py +++ b/modules/features/neutralization/datamodelFeatureNeutralizer.py @@ -3,17 +3,33 @@ """Neutralizer models: DataNeutraliserConfig and DataNeutralizerAttributes.""" import uuid +from enum import Enum from typing import Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -class DataNeutraliserConfig(BaseModel): +class DataScope(str, Enum): + PERSONAL = "personal" + FEATURE_INSTANCE = "featureInstance" + MANDATE = "mandate" + GLOBAL = "global" + + +class DataNeutraliserConfig(PowerOnModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) mandateId: str = Field(description="ID of the mandate this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) featureInstanceId: str = Field(description="ID of the feature instance this configuration belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) userId: str = Field(description="ID of the user who created this configuration", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) enabled: bool = Field(default=True, description="Whether data neutralization is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) + scope: str = Field(default="personal", description="Data visibility scope: personal, featureInstance, mandate, global", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ + {"value": "personal", "label": {"en": "Personal", "de": "Persönlich"}}, + {"value": "featureInstance", "label": {"en": "Feature Instance", "de": "Feature-Instanz"}}, + {"value": "mandate", "label": {"en": "Mandate", "de": "Mandant"}}, + {"value": "global", "label": {"en": "Global", "de": "Global"}}, + ]}) + neutralizationStatus: str = Field(default="not_required", description="Status of neutralization: pending, completed, failed, not_required", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) namesToParse: str = Field(default="", description="Multiline list of names to parse for neutralization", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False}) sharepointSourcePath: str = Field(default="", description="SharePoint path to read files for neutralization", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) sharepointTargetPath: str = Field(default="", description="SharePoint path to store neutralized files", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) @@ -26,6 +42,8 @@ registerModelLabels( "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, "userId": {"en": "User ID", "fr": "ID utilisateur"}, "enabled": {"en": "Enabled", "fr": "Activé"}, + "scope": {"en": "Scope", "fr": "Portée"}, + "neutralizationStatus": {"en": "Neutralization Status", "fr": "Statut de neutralisation"}, "namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"}, "sharepointSourcePath": {"en": "Source Path", "fr": "Chemin source"}, "sharepointTargetPath": {"en": "Target Path", "fr": "Chemin cible"}, @@ -40,6 +58,17 @@ class DataNeutralizerAttributes(BaseModel): originalText: str = Field(description="Original text that was neutralized", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) fileId: Optional[str] = Field(default=None, description="ID of the file this attribute belongs to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) patternType: str = Field(description="Type of pattern that matched (email, phone, name, etc.)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) + + +class DataNeutralizationSnapshot(BaseModel): + """Stores the full neutralized text (with embedded placeholders) per source.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + mandateId: str = Field(description="Mandate scope") + featureInstanceId: str = Field(default="", description="Feature instance scope") + userId: str = Field(description="User who triggered neutralization") + sourceLabel: str = Field(description="Human label, e.g. 'Prompt', 'Kontext', 'Nachricht 3'") + neutralizedText: str = Field(description="Full text with [type.uuid] placeholders embedded") + placeholderCount: int = Field(default=0, description="Number of placeholders in the text") registerModelLabels( "DataNeutralizerAttributes", {"en": "Neutralized Data Attribute", "fr": "Attribut de données neutralisées"}, @@ -53,5 +82,18 @@ registerModelLabels( "patternType": {"en": "Pattern Type", "fr": "Type de modèle"}, }, ) +registerModelLabels( + "DataNeutralizationSnapshot", + {"en": "Neutralization Snapshot", "de": "Neutralisierungs-Snapshot"}, + { + "id": {"en": "ID"}, + "mandateId": {"en": "Mandate ID"}, + "featureInstanceId": {"en": "Feature Instance ID"}, + "userId": {"en": "User ID"}, + "sourceLabel": {"en": "Source", "de": "Quelle"}, + "neutralizedText": {"en": "Neutralized Text", "de": "Neutralisierter Text"}, + "placeholderCount": {"en": "Placeholders", "de": "Platzhalter"}, + }, +) diff --git a/modules/features/neutralization/interfaceFeatureNeutralizer.py b/modules/features/neutralization/interfaceFeatureNeutralizer.py index 54e3e368..22af9683 100644 --- a/modules/features/neutralization/interfaceFeatureNeutralizer.py +++ b/modules/features/neutralization/interfaceFeatureNeutralizer.py @@ -11,6 +11,7 @@ from typing import Dict, List, Any, Optional from modules.features.neutralization.datamodelFeatureNeutralizer import ( DataNeutraliserConfig, DataNeutralizerAttributes, + DataNeutralizationSnapshot, ) from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.interfaces.interfaceRbac import getRecordsetWithRBAC @@ -212,6 +213,89 @@ class InterfaceFeatureNeutralizer: logger.error(f"Error getting attribute by ID: {str(e)}") return None + def deleteAttributeById(self, attributeId: str) -> bool: + """Delete a single neutralization attribute by its ID""" + try: + attribute = self.getAttributeById(attributeId) + if not attribute: + logger.warning(f"Attribute {attributeId} not found for deletion") + return False + + self.db.recordDelete(DataNeutralizerAttributes, attributeId) + logger.info(f"Deleted neutralization attribute {attributeId}") + return True + except Exception as e: + logger.error(f"Error deleting attribute by ID: {str(e)}") + return False + + # ------------------------------------------------------------------ + # Snapshot CRUD + # ------------------------------------------------------------------ + + def getSnapshots(self) -> List[DataNeutralizationSnapshot]: + """Return all neutralization snapshots for the current mandate + feature instance.""" + try: + _filter: Dict[str, Any] = {"mandateId": self.mandateId} + if self.featureInstanceId: + _filter["featureInstanceId"] = self.featureInstanceId + rows = getRecordsetWithRBAC( + self.db, + DataNeutralizationSnapshot, + self.currentUser, + recordFilter=_filter, + mandateId=self.mandateId, + ) + return [ + DataNeutralizationSnapshot(**{k: v for k, v in r.items() if not k.startswith("_")}) + for r in rows + ] + except Exception as e: + logger.error(f"Error getting snapshots: {e}") + return [] + + def clearSnapshots(self) -> int: + """Delete all snapshots for the current feature-instance scope. Returns count deleted.""" + try: + _filter: Dict[str, Any] = {"mandateId": self.mandateId} + if self.featureInstanceId: + _filter["featureInstanceId"] = self.featureInstanceId + existing = self.db.getRecordset(DataNeutralizationSnapshot, recordFilter=_filter) + for row in existing: + self.db.recordDelete(DataNeutralizationSnapshot, row["id"]) + return len(existing) + except Exception as e: + logger.error(f"Error clearing snapshots: {e}") + return 0 + + def createSnapshot( + self, + sourceLabel: str, + neutralizedText: str, + placeholderCount: int = 0, + ) -> Optional[DataNeutralizationSnapshot]: + """Persist one neutralization snapshot.""" + try: + if not self.userId: + logger.warning("Cannot create snapshot: missing userId") + return None + snap = DataNeutralizationSnapshot( + mandateId=self.mandateId or "", + featureInstanceId=self.featureInstanceId or "", + userId=self.userId, + sourceLabel=sourceLabel, + neutralizedText=neutralizedText, + placeholderCount=placeholderCount, + ) + created = self.db.recordCreate(DataNeutralizationSnapshot, snap.model_dump()) + return DataNeutralizationSnapshot(**{k: v for k, v in created.items() if not k.startswith("_")}) + except Exception as e: + logger.error(f"Error creating snapshot: {e}") + return None + + # ------------------------------------------------------------------ + # Attribute CRUD + # ------------------------------------------------------------------ + def createAttribute( self, attributeId: str, diff --git a/modules/features/neutralization/mainNeutralization.py b/modules/features/neutralization/mainNeutralization.py index d32b441f..bfe97a13 100644 --- a/modules/features/neutralization/mainNeutralization.py +++ b/modules/features/neutralization/mainNeutralization.py @@ -45,34 +45,55 @@ RESOURCE_OBJECTS = [ # Template roles for this feature TEMPLATE_ROLES = [ + { + "roleLabel": "neutralization-viewer", + "description": { + "en": "Neutralization Viewer - View neutralization data (read-only)", + "de": "Neutralisierungs-Betrachter - Neutralisierungsdaten einsehen (nur lesen)", + "fr": "Visualiseur neutralisation - Consulter les données de neutralisation (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.neutralization.playground", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, + { + "roleLabel": "neutralization-user", + "description": { + "en": "Neutralization User - Use neutralization tools and manage own data", + "de": "Neutralisierungs-Benutzer - Neutralisierungstools nutzen und eigene Daten verwalten", + "fr": "Utilisateur neutralisation - Utiliser les outils et gérer ses propres données", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.neutralization.playground", "view": True}, + {"context": "UI", "item": "ui.feature.neutralization.attributes", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + ], + }, { "roleLabel": "neutralization-admin", "description": { "en": "Neutralization Administrator - Full access to neutralization settings and data", "de": "Neutralisierungs-Administrator - Vollzugriff auf Neutralisierungs-Einstellungen und Daten", - "fr": "Administrateur neutralisation - Accès complet aux paramètres et données" + "fr": "Administrateur neutralisation - Accès complet aux paramètres et données", }, "accessRules": [ - # Full UI access (all views including admin views) {"context": "UI", "item": None, "view": True}, - # Full DATA access {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, - ] + ], }, { "roleLabel": "neutralization-analyst", "description": { "en": "Neutralization Analyst - Analyze and process neutralization data", "de": "Neutralisierungs-Analyst - Neutralisierungsdaten analysieren und verarbeiten", - "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation" + "fr": "Analyste neutralisation - Analyser et traiter les données de neutralisation", }, "accessRules": [ - # UI access to specific views - vollqualifizierte ObjectKeys {"context": "UI", "item": "ui.feature.neutralization.playground", "view": True}, {"context": "UI", "item": "ui.feature.neutralization.attributes", "view": True}, - # Group-level DATA access (read-only for sensitive config) {"context": "DATA", "item": None, "view": True, "read": "g", "create": "n", "update": "n", "delete": "n"}, - ] + ], }, ] diff --git a/modules/features/neutralization/neutralizePlayground.py b/modules/features/neutralization/neutralizePlayground.py index c92d241b..500cc1ba 100644 --- a/modules/features/neutralization/neutralizePlayground.py +++ b/modules/features/neutralization/neutralizePlayground.py @@ -6,7 +6,8 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlparse, unquote from modules.datamodels.datamodelUam import User -from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig +from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig, DataNeutralizationSnapshot +from .interfaceFeatureNeutralizer import getInterface as _getNeutralizerInterface from modules.serviceHub import getInterface as getServices logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ class NeutralizationPlayground: 'neutralized_file_id': None, 'processed_info': {'type': 'error', 'error': 'File could not be decoded as text. Supported: UTF-8, Latin-1. For PDF/Word/Excel, use supported binary formats.'} } - result = self.services.neutralization.processText(text_content) + result = await self.services.neutralization.processTextAsync(text_content) result['neutralized_file_name'] = f'neutralized_{filename}' # Save neutralized text as file to user files if self.services.interfaceDbComponent and result.get('neutralized_text') is not None: @@ -129,6 +130,11 @@ class NeutralizationPlayground: } + # Delete a single attribute by ID + def deleteAttribute(self, attributeId: str) -> bool: + interface = _getNeutralizerInterface(self.currentUser, self.mandateId, self.featureInstanceId) + return interface.deleteAttributeById(attributeId) + # Cleanup attributes def cleanAttributes(self, fileId: str) -> bool: return self.services.neutralization.deleteNeutralizationAttributes(fileId) @@ -192,12 +198,28 @@ class NeutralizationPlayground: """Resolve UIDs in neutralized text back to original text""" return self.services.neutralization.resolveText(text) + def getSnapshots(self) -> List[DataNeutralizationSnapshot]: + """Return stored neutralization text snapshots.""" + try: + return self.services.neutralization.getSnapshots() + except Exception as e: + logger.error(f"Error getting snapshots: {e}") + return [] + def getAttributes(self, fileId: str = None) -> List[DataNeutralizerAttributes]: """Get neutralization attributes, optionally filtered by file ID""" try: allAttributes = self.services.neutralization.getAttributes() if fileId: - return [attr for attr in allAttributes if attr.fileId == fileId] + want = str(fileId).strip() + + def _matches(a: DataNeutralizerAttributes) -> bool: + af = a.fileId + if af is None or (isinstance(af, str) and not str(af).strip()): + return False + return str(af).strip() == want + + return [attr for attr in allAttributes if _matches(attr)] return allAttributes except Exception as e: logger.error(f"Error getting attributes: {str(e)}") @@ -390,7 +412,7 @@ class SharepointProcessor: textContent = fileContent.decode('utf-8') except UnicodeDecodeError: textContent = fileContent.decode('latin-1') - result = self.services.neutralization.processText(textContent) + result = await self.services.neutralization.processTextAsync(textContent) content_to_upload = (result.get('neutralized_text') or '').encode('utf-8') neutralizedFilename = f"neutralized_{fileInfo['name']}" diff --git a/modules/features/neutralization/routeFeatureNeutralizer.py b/modules/features/neutralization/routeFeatureNeutralizer.py index de49f50d..2f36efef 100644 --- a/modules/features/neutralization/routeFeatureNeutralizer.py +++ b/modules/features/neutralization/routeFeatureNeutralizer.py @@ -8,12 +8,33 @@ import logging from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces -from .datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes +from .datamodelFeatureNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes, DataNeutralizationSnapshot from .neutralizePlayground import NeutralizationPlayground # Configure logger logger = logging.getLogger(__name__) + +def _assertFeatureInstancePathMatchesContext(featureInstanceIdFromPath: str, context: RequestContext) -> None: + """Reject path/instance mismatch when request context already carries an instance id.""" + ctxId = str(context.featureInstanceId).strip() if getattr(context, "featureInstanceId", None) else "" + pathId = (featureInstanceIdFromPath or "").strip() + if ctxId and pathId and pathId != ctxId: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Feature instance id in URL does not match request context (X-Instance-Id)", + ) + + +def _fetchNeutralizationAttributes(context: RequestContext, fileId: Optional[str]) -> List[DataNeutralizerAttributes]: + service = NeutralizationPlayground( + context.user, + str(context.mandateId) if context.mandateId else "", + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + return service.getAttributes(fileId) + + # Create router for neutralization endpoints router = APIRouter( prefix="/api/neutralization", @@ -208,15 +229,9 @@ def get_neutralization_attributes( ) -> List[DataNeutralizerAttributes]: """Get neutralization attributes, optionally filtered by file ID""" try: - service = NeutralizationPlayground( - context.user, - str(context.mandateId) if context.mandateId else "", - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None - ) - attributes = service.getAttributes(fileId) - - return attributes - + return _fetchNeutralizationAttributes(context, fileId) + except HTTPException: + raise except Exception as e: logger.error(f"Error getting neutralization attributes: {str(e)}") raise HTTPException( @@ -224,6 +239,72 @@ def get_neutralization_attributes( detail=f"Error getting neutralization attributes: {str(e)}" ) + +@router.get("/{feature_instance_id}/attributes", response_model=List[DataNeutralizerAttributes]) +@limiter.limit("30/minute") +def get_neutralization_attributes_scoped( + request: Request, + feature_instance_id: str = Path(..., description="Workspace / feature instance id (must match X-Instance-Id when set)"), + fileId: Optional[str] = Query(None, description="Filter by file ID"), + context: RequestContext = Depends(getRequestContext), +) -> List[DataNeutralizerAttributes]: + """Same as GET /attributes; path includes instance id for workspace UI compatibility.""" + _assertFeatureInstancePathMatchesContext(feature_instance_id, context) + try: + return _fetchNeutralizationAttributes(context, fileId) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting neutralization attributes: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error getting neutralization attributes: {str(e)}" + ) + +@router.get("/snapshots", response_model=List[DataNeutralizationSnapshot]) +@limiter.limit("30/minute") +def get_neutralization_snapshots( + request: Request, + context: RequestContext = Depends(getRequestContext), +) -> List[DataNeutralizationSnapshot]: + """Return neutralized-text snapshots (full text with placeholders) for the current feature instance.""" + try: + service = NeutralizationPlayground( + context.user, + str(context.mandateId) if context.mandateId else "", + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + return service.getSnapshots() + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting neutralization snapshots: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + +@router.get("/{feature_instance_id}/snapshots", response_model=List[DataNeutralizationSnapshot]) +@limiter.limit("30/minute") +def get_neutralization_snapshots_scoped( + request: Request, + feature_instance_id: str = Path(..., description="Workspace instance id (must match X-Instance-Id when set)"), + context: RequestContext = Depends(getRequestContext), +) -> List[DataNeutralizationSnapshot]: + """Same as GET /snapshots; path includes instance id for workspace UI (explicit scope).""" + _assertFeatureInstancePathMatchesContext(feature_instance_id, context) + try: + service = NeutralizationPlayground( + context.user, + str(context.mandateId) if context.mandateId else "", + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + return service.getSnapshots() + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting neutralization snapshots (scoped): {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.post("/process-sharepoint", response_model=Dict[str, Any]) @limiter.limit("5/minute") async def process_sharepoint_files( @@ -317,6 +398,108 @@ def get_neutralization_stats( detail=f"Error getting neutralization stats: {str(e)}" ) +def _deleteSingleNeutralizationAttribute(context: RequestContext, attributeId: str) -> Dict[str, str]: + service = NeutralizationPlayground( + context.user, + str(context.mandateId) if context.mandateId else "", + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + success = service.deleteAttribute(attributeId) + if success: + return {"message": f"Attribute {attributeId} deleted"} + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Attribute {attributeId} not found", + ) + + +@router.delete("/attributes/single/{attributeId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +def deleteAttribute( + request: Request, + attributeId: str = Path(..., description="Attribute ID to delete"), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, str]: + """Delete a single neutralization attribute by ID.""" + try: + return _deleteSingleNeutralizationAttribute(context, attributeId) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting attribute: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.delete("/{feature_instance_id}/attributes/single/{attributeId}", response_model=Dict[str, str]) +@limiter.limit("30/minute") +def deleteAttributeScoped( + request: Request, + feature_instance_id: str = Path(..., description="Workspace / feature instance id"), + attributeId: str = Path(..., description="Attribute ID to delete"), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, str]: + """Same as DELETE /attributes/single/{attributeId}; path includes instance id for workspace UI.""" + _assertFeatureInstancePathMatchesContext(feature_instance_id, context) + try: + return _deleteSingleNeutralizationAttribute(context, attributeId) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting attribute: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +def _retriggerNeutralizationBody(context: RequestContext, fileId: str) -> Dict[str, str]: + if not fileId: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="fileId is required", + ) + service = NeutralizationPlayground( + context.user, + str(context.mandateId) if context.mandateId else "", + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + service.cleanupFileAttributes(fileId) + return {"message": f"Neutralization re-triggered for file {fileId}", "fileId": fileId} + + +@router.post("/retrigger", response_model=Dict[str, str]) +@limiter.limit("10/minute") +def retriggerNeutralization( + request: Request, + retriggerData: Dict[str, str] = Body(...), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, str]: + """Re-trigger neutralization for a specific file.""" + try: + return _retriggerNeutralizationBody(context, retriggerData.get("fileId", "")) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error re-triggering neutralization: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/{feature_instance_id}/retrigger", response_model=Dict[str, str]) +@limiter.limit("10/minute") +def retriggerNeutralizationScoped( + request: Request, + feature_instance_id: str = Path(..., description="Workspace / feature instance id"), + retriggerData: Dict[str, str] = Body(...), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, str]: + """Same as POST /retrigger; path includes instance id for workspace UI compatibility.""" + _assertFeatureInstancePathMatchesContext(feature_instance_id, context) + try: + return _retriggerNeutralizationBody(context, retriggerData.get("fileId", "")) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error re-triggering neutralization: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.delete("/attributes/{fileId}", response_model=Dict[str, str]) @limiter.limit("10/minute") def cleanup_file_attributes( diff --git a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py index c803b375..4c0842d4 100644 --- a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py @@ -60,6 +60,12 @@ class NeutralizationService: mandateId=serviceCenter.mandateId or dbApp.mandateId, featureInstanceId=getattr(serviceCenter, 'featureInstanceId', None) or getattr(dbApp, 'featureInstanceId', None) ) + elif serviceCenter and getattr(serviceCenter, "user", None): + self.interfaceNeutralizer = getNeutralizerInterface( + currentUser=serviceCenter.user, + mandateId=getattr(serviceCenter, 'mandateId', None) or getattr(serviceCenter, 'mandate_id', None), + featureInstanceId=getattr(serviceCenter, 'featureInstanceId', None) or getattr(serviceCenter, 'feature_instance_id', None), + ) namesList = NamesToParse if isinstance(NamesToParse, list) else [] self.NamesToParse = namesList @@ -82,11 +88,213 @@ class NeutralizationService: # Public API: process text or file - def processText(self, text: str) -> Dict[str, Any]: - """Neutralize a raw text string and return a standard result dict.""" - result = self._neutralizeText(text, 'text') - self._persistAttributes(result.get('mapping', {}), None) - return result + _NEUT_INSTRUCTION = ( + "Analyze the following text and identify ALL sensitive content that must be neutralized:\n" + "1. Personal data (PII): names of persons, email addresses, phone numbers, " + "physical addresses, ID numbers, dates of birth, financial data (IBAN, account numbers), " + "social security numbers\n" + "2. Protected business logic: proprietary algorithms, trade secrets, confidential " + "processes, internal procedures, code snippets that reveal implementation details\n" + "3. Named entities: company names, product names, project names, brand names\n\n" + "Return ONLY a JSON array (no markdown, no explanation):\n" + '[{"text":"exact substring","type":"name|email|phone|address|id|financial|logic|company|product|location|other"}]\n\n' + "Rules:\n" + "- Every entry's 'text' must be an exact, verbatim substring of the input.\n" + "- Do NOT include generic words, common language constructs or non-sensitive terms.\n" + "- If nothing is sensitive, return [].\n\n" + ) + _BYTES_PER_TOKEN = 3 + _SELECTOR_MAX_RATIO = 0.8 + _CHUNK_SAFETY_MARGIN = 0.9 + + def _resolveNeutModel(self): + """Query the model registry for the best NEUTRALIZATION_TEXT model. + Returns the model object (with contextLength etc.) or None.""" + try: + from modules.aicore.aicoreModelRegistry import modelRegistry + from modules.aicore.aicoreModelSelector import modelSelector as _modSel + from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum + + _models = modelRegistry.getAvailableModels() + _opts = AiCallOptions(operationType=OperationTypeEnum.NEUTRALIZATION_TEXT) + _failover = _modSel.getFailoverModelList("x", "", _opts, _models) + return _failover[0] if _failover else None + except Exception as _e: + logger.warning(f"_resolveNeutModel failed: {_e}") + return None + + def _calcMaxChunkChars(self, model) -> int: + """Derive the maximum text-chunk size (in characters) from the selected + model's contextLength, mirroring the rules in aicoreModelSelector: + promptTokens = promptBytes / 3 must be <= contextLength * 0.8 + Subtract the instruction overhead and apply a safety margin.""" + if not model or getattr(model, 'contextLength', 0) <= 0: + return 5000 + _instructionBytes = len(self._NEUT_INSTRUCTION.encode('utf-8')) + 30 + _maxPromptBytes = int(model.contextLength * self._SELECTOR_MAX_RATIO * self._BYTES_PER_TOKEN) + _maxChunkChars = int((_maxPromptBytes - _instructionBytes) * self._CHUNK_SAFETY_MARGIN) + return max(_maxChunkChars, 500) + + @staticmethod + def _splitTextIntoChunks(text: str, maxChars: int) -> List[str]: + """Split *text* into chunks of at most *maxChars*, preferring paragraph + then sentence boundaries so that the LLM sees coherent blocks.""" + if len(text) <= maxChars: + return [text] + + chunks: List[str] = [] + remaining = text + while remaining: + if len(remaining) <= maxChars: + chunks.append(remaining) + break + _cut = maxChars + _para = remaining.rfind("\n\n", 0, _cut) + if _para > maxChars // 3: + _cut = _para + 2 + else: + _nl = remaining.rfind("\n", 0, _cut) + if _nl > maxChars // 3: + _cut = _nl + 1 + else: + _dot = remaining.rfind(". ", 0, _cut) + if _dot > maxChars // 3: + _cut = _dot + 2 + else: + _sp = remaining.rfind(" ", 0, _cut) + if _sp > maxChars // 3: + _cut = _sp + 1 + chunks.append(remaining[:_cut]) + remaining = remaining[_cut:] + return chunks + + async def _analyseChunk(self, aiService, chunkText: str) -> List[dict]: + """Send one chunk to the NEUTRALIZATION_TEXT model, return raw findings list.""" + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + + _prompt = self._NEUT_INSTRUCTION + "Text to analyze:\n---\n" + chunkText + "\n---" + _request = AiCallRequest( + prompt=_prompt, + options=AiCallOptions(operationType=OperationTypeEnum.NEUTRALIZATION_TEXT), + ) + _response = await aiService.callAi(_request) + if not _response or not getattr(_response, 'content', None): + raise RuntimeError( + "Neutralization AI call returned no response " + "(no model available for NEUTRALIZATION_TEXT?)" + ) + if getattr(_response, 'errorCount', 0) > 0 or getattr(_response, 'modelName', '') == 'error': + raise RuntimeError( + f"Neutralization AI call failed: {_response.content}" + ) + _content = _response.content.strip() + if _content.startswith("```"): + _content = _content.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + return json.loads(_content) + except json.JSONDecodeError: + _bracket = _content.find("[") + if _bracket >= 0: + try: + return json.loads(_content[_bracket:]) + except json.JSONDecodeError: + pass + return [] + + async def processTextAsync(self, text: str, fileId: Optional[str] = None) -> Dict[str, Any]: + """AI-powered text neutralization with automatic chunking. + + If *text* exceeds the safe token budget for the neutralization model + it is split into smaller chunks, each analysed separately. Findings + are merged and de-duplicated before placeholder replacement. + + Regex patterns run as a supplementary pass to catch anything the + model missed. + """ + import uuid as _uuid + + aiService = None + if self._getService: + try: + aiService = self._getService("ai") + except Exception: + pass + + aiMapping: Dict[str, str] = {} + + if not aiService or not hasattr(aiService, 'callAi'): + raise RuntimeError("Neutralization requires an AI service but none is available") + + if text.strip(): + _neutModel = self._resolveNeutModel() + _maxChunkChars = self._calcMaxChunkChars(_neutModel) + logger.info( + f"processTextAsync: model={getattr(_neutModel, 'name', '?')}, " + f"contextLength={getattr(_neutModel, 'contextLength', '?')} tokens, " + f"maxChunkChars={_maxChunkChars}" + ) + + _chunks = self._splitTextIntoChunks(text, _maxChunkChars) + if len(_chunks) > 1: + logger.info( + f"processTextAsync: text ({len(text)} chars) " + f"split into {len(_chunks)} chunk(s) of max {_maxChunkChars} chars" + ) + + for _chunkIdx, _chunkText in enumerate(_chunks): + _findings = await self._analyseChunk(aiService, _chunkText) + if not isinstance(_findings, list): + continue + for _f in _findings: + if not isinstance(_f, dict): + continue + _origText = _f.get("text", "") + _patType = _f.get("type", "other").lower() + if not _origText or _origText not in text: + continue + if _origText in aiMapping: + continue + _uid = str(_uuid.uuid4()) + _placeholder = f"[{_patType}.{_uid}]" + aiMapping[_origText] = _placeholder + + logger.info(f"AI neutralization found {len(aiMapping)} item(s)" + + (f" across {len(_chunks)} chunk(s)" if len(_chunks) > 1 else "")) + + neutralizedText = text + for _orig, _ph in sorted(aiMapping.items(), key=lambda x: -len(x[0])): + neutralizedText = neutralizedText.replace(_orig, _ph) + + regexMapping: Dict[str, str] = {} + finalText = neutralizedText + + allMapping = {**aiMapping, **regexMapping} + if allMapping: + _loop = asyncio.get_event_loop() + await _loop.run_in_executor( + None, self._persistAttributes, allMapping, fileId + ) + logger.debug(f"processTextAsync: {len(allMapping)} attribute(s) persisted") + + return { + 'neutralized_text': finalText, + 'mapping': allMapping, + 'attributes': [ + NeutralizationAttribute(original=k, placeholder=v) + for k, v in allMapping.items() + ], + 'processed_info': {'type': 'text', 'ai_findings': len(aiMapping), 'regex_findings': len(regexMapping)}, + } + + def processText(self, text: str, fileId: Optional[str] = None) -> Dict[str, Any]: + """Sync wrapper around processTextAsync. Propagates errors.""" + try: + return asyncio.run(self.processTextAsync(text, fileId)) + except RuntimeError as _re: + if "cannot be called from a running event loop" in str(_re): + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.processTextAsync(text, fileId)) + raise def processFile(self, fileId: str) -> Dict[str, Any]: """Neutralize a file referenced by its fileId using component interface. @@ -153,8 +361,7 @@ class NeutralizationService: raise ValueError("Unable to decode file content as text.") textContent = decoded - result = self._neutralizeText(textContent, textType) - self._persistAttributes(result.get('mapping', {}), fileId) + result = self.processText(textContent, fileId) if fileName: result['neutralized_file_name'] = f"neutralized_{fileName}" result['file_id'] = fileId @@ -203,6 +410,89 @@ class NeutralizationService: 'processed_info': {'type': 'binary', 'status': 'error', 'error': str(e)} } + async def processImageAsync(self, imageBytes: bytes, fileName: str, mimeType: str = "image/png") -> Dict[str, Any]: + """Analyze image via internal vision model to check for sensitive content. + + Returns dict with: + - 'status': 'ok' | 'blocked' | 'error' + - 'hasSensitiveContent': bool + - 'analysis': str (model's analysis text, if available) + - 'processed_info': dict with details + + Uses NEUTRALIZATION_IMAGE operation type → only internal Private-LLM models. + If no internal model available → returns 'blocked'. + """ + import base64 + try: + aiService = None + if self._getService: + try: + aiService = self._getService("ai") + except Exception: + pass + if not aiService or not hasattr(aiService, 'callAi'): + logger.warning(f"processImage: AI service not available — blocking image '{fileName}'") + return { + 'status': 'blocked', + 'hasSensitiveContent': True, + 'analysis': '', + 'processed_info': {'type': 'image', 'status': 'blocked', 'reason': 'AI service unavailable'} + } + + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + + _b64Data = base64.b64encode(imageBytes).decode('utf-8') + _dataUrl = f"data:{mimeType};base64,{_b64Data}" + + _prompt = ( + "Analyze this image for personally identifiable information (PII). " + "Check for: names, addresses, phone numbers, email addresses, ID numbers, " + "faces, signatures, handwritten text, license plates, financial data. " + "Respond with JSON: {\"hasPII\": true/false, \"findings\": [\"...\"]}" + ) + + _request = AiCallRequest( + prompt=_prompt, + options=AiCallOptions(operationType=OperationTypeEnum.NEUTRALIZATION_IMAGE), + messages=[{"role": "user", "content": [ + {"type": "text", "text": _prompt}, + {"type": "image_url", "image_url": {"url": _dataUrl}}, + ]}], + ) + + _response = await aiService.callAi(_request) + + _hasPII = False + _analysis = _response.content if _response and hasattr(_response, 'content') else '' + if _analysis: + _lowerAnalysis = _analysis.lower() + if '"haspii": true' in _lowerAnalysis or '"haspii":true' in _lowerAnalysis: + _hasPII = True + + return { + 'status': 'blocked' if _hasPII else 'ok', + 'hasSensitiveContent': _hasPII, + 'analysis': _analysis, + 'processed_info': {'type': 'image', 'status': 'blocked' if _hasPII else 'ok', 'fileName': fileName} + } + except Exception as e: + logger.error(f"processImage failed for '{fileName}': {e}") + return { + 'status': 'blocked', + 'hasSensitiveContent': True, + 'analysis': '', + 'processed_info': {'type': 'image', 'status': 'error', 'error': str(e)} + } + + def processImage(self, imageBytes: bytes, fileName: str, mimeType: str = "image/png") -> Dict[str, Any]: + """Sync wrapper for processImageAsync. Uses asyncio.run when no event loop is running.""" + import asyncio + try: + return asyncio.run(self.processImageAsync(imageBytes, fileName, mimeType)) + except RuntimeError: + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.processImageAsync(imageBytes, fileName, mimeType)) + def resolveText(self, text: str) -> str: if not self.interfaceNeutralizer: return text @@ -236,6 +526,22 @@ class NeutralizationService: return False return self.interfaceNeutralizer.deleteNeutralizationAttributes(fileId) + def getSnapshots(self): + if not self.interfaceNeutralizer: + return [] + return self.interfaceNeutralizer.getSnapshots() + + def clearSnapshots(self) -> int: + if not self.interfaceNeutralizer: + return 0 + return self.interfaceNeutralizer.clearSnapshots() + + def saveSnapshot(self, sourceLabel: str, neutralizedText: str, placeholderCount: int = 0): + if not self.interfaceNeutralizer: + logger.warning("saveSnapshot: interfaceNeutralizer is None — snapshot not stored") + return None + return self.interfaceNeutralizer.createSnapshot(sourceLabel, neutralizedText, placeholderCount) + def _persistAttributes(self, mapping: Dict[str, str], fileId: Optional[str]) -> None: """Persist mapping to DB for resolve to work. mapping: originalText -> placeholder e.g. '[email.uuid]'""" if not self.interfaceNeutralizer or not mapping: @@ -295,10 +601,22 @@ class NeutralizationService: p = part if isinstance(part, dict) else part.model_dump() if hasattr(part, 'model_dump') else part type_group = p.get('typeGroup', '') data = p.get('data', '') - if type_group in ('binary', 'image') or not (data and str(data).strip()): + if type_group == 'binary' or not (data and str(data).strip()): neutralized_parts.append(part) continue - nr = self._neutralizeText(str(data), 'text' if type_group != 'table' else 'csv') + if type_group == 'image': + import base64 as _b64img + try: + _imgBytes = _b64img.b64decode(str(data)) + _imgResult = await self.processImageAsync(_imgBytes, fileName) + if _imgResult.get("status") == "ok": + neutralized_parts.append(part) + else: + logger.warning(f"Image part blocked in binary file '{fileName}' (PII detected), removing") + except Exception as _imgErr: + logger.warning(f"Image check failed in binary file '{fileName}': {_imgErr}, removing (fail-safe)") + continue + nr = await self.processTextAsync(str(data), fileId) proc = nr.get('processed_info', {}) or {} if isinstance(proc, dict) and proc.get('type') == 'error': neutralization_error = proc.get('error', 'Neutralization failed') @@ -307,7 +625,6 @@ class NeutralizationService: all_mapping.update(mapping) new_part = {**p, 'data': neu_text} neutralized_parts.append(new_part) - self._persistAttributes(all_mapping, fileId) # 3. PDF: Use in-place only; no fallback to render if mimeType == "application/pdf": @@ -451,10 +768,31 @@ class NeutralizationService: # Helper functions + def _neutralizeTextLight(self, text: str) -> Dict[str, Any]: + """Regex-only supplementary pass using already-initialised processors. + + Unlike ``_neutralizeText`` this does **no** DB I/O + (``_reloadNamesFromConfig`` is skipped) so it is safe to call from + an async context without blocking the event-loop or risking a + DB-connection-pool deadlock during parallel document processing. + """ + try: + data, mapping, replaced_fields, processed_info = self.textProcessor.processTextContent(text) + neutralized_text = str(data) + attributes = [NeutralizationAttribute(original=k, placeholder=v) for k, v in mapping.items()] + return NeutralizationResult( + neutralized_text=neutralized_text, + mapping=mapping, + attributes=attributes, + processed_info=processed_info, + ).model_dump() + except Exception as e: + logger.warning(f"_neutralizeTextLight error: {e}") + return {'neutralized_text': text, 'mapping': {}, 'attributes': [], 'processed_info': {'type': 'error', 'error': str(e)}} + def _neutralizeText(self, text: str, textType: str = None) -> Dict[str, Any]: """Process text and return unified dict for API consumption.""" try: - # Reload names from config before processing to ensure we have the latest names self._reloadNamesFromConfig() # Auto-detect content type if not provided diff --git a/modules/features/realEstate/datamodelFeatureRealEstate.py b/modules/features/realEstate/datamodelFeatureRealEstate.py index 31efbc07..8f136056 100644 --- a/modules/features/realEstate/datamodelFeatureRealEstate.py +++ b/modules/features/realEstate/datamodelFeatureRealEstate.py @@ -7,6 +7,7 @@ Implements a general Swiss architecture planning data model. from typing import List, Dict, Any, Optional, ForwardRef from enum import Enum from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp import uuid @@ -178,7 +179,7 @@ class Dokument(BaseModel): ) -class Kontext(BaseModel): +class Kontext(PowerOnModel): """Supporting data object for flexible additional information.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -248,7 +249,7 @@ class Land(BaseModel): ) -class Kanton(BaseModel): +class Kanton(PowerOnModel): """Cantonal level administrative entity.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -368,7 +369,7 @@ class Gemeinde(BaseModel): ParzelleRef = ForwardRef('Parzelle') -class Parzelle(BaseModel): +class Parzelle(PowerOnModel): """Represents a plot with all building law properties.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -594,7 +595,7 @@ class Parzelle(BaseModel): ) -class Projekt(BaseModel): +class Projekt(PowerOnModel): """Core object representing a construction project.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 2ae2378b..dfe310d5 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -39,52 +39,57 @@ RESOURCE_OBJECTS = [ # Template roles for this feature with AccessRules # IMPORTANT: item uses vollqualifizierte ObjectKeys (gemäss Navigation-API-Konzept) TEMPLATE_ROLES = [ + { + "roleLabel": "realestate-viewer", + "description": { + "en": "Real Estate Viewer - View property information (read-only)", + "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen (nur lesen)", + "fr": "Visualiseur immobilier - Consulter les informations immobilières (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.realestate.dashboard", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, + { + "roleLabel": "realestate-user", + "description": { + "en": "Real Estate User - Create and manage own property records", + "de": "Immobilien-Benutzer - Eigene Immobilien-Daten erstellen und verwalten", + "fr": "Utilisateur immobilier - Créer et gérer ses propres données immobilières", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.realestate.dashboard", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + {"context": "RESOURCE", "item": "resource.feature.realestate.project.create", "view": True}, + ], + }, { "roleLabel": "realestate-admin", "description": { "en": "Real Estate Administrator - Full access to all property data and settings", "de": "Immobilien-Administrator - Vollzugriff auf alle Immobiliendaten und Einstellungen", - "fr": "Administrateur immobilier - Accès complet aux données et paramètres" + "fr": "Administrateur immobilier - Accès complet aux données et paramètres", }, "accessRules": [ - # Full UI access (all views including admin views) {"context": "UI", "item": None, "view": True}, - # Full DATA access {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, - # Admin resources {"context": "RESOURCE", "item": "resource.feature.realestate.project.create", "view": True}, {"context": "RESOURCE", "item": "resource.feature.realestate.project.delete", "view": True}, - ] + ], }, { "roleLabel": "realestate-manager", "description": { "en": "Real Estate Manager - Manage properties and tenants", "de": "Immobilien-Verwalter - Immobilien und Mieter verwalten", - "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires" + "fr": "Gestionnaire immobilier - Gérer les propriétés et locataires", }, "accessRules": [ - # UI access to map view {"context": "UI", "item": "ui.feature.realestate.dashboard", "view": True}, - # Group-level DATA access {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, - # Resource: create projects {"context": "RESOURCE", "item": "resource.feature.realestate.project.create", "view": True}, - ] - }, - { - "roleLabel": "realestate-viewer", - "description": { - "en": "Real Estate Viewer - View property information", - "de": "Immobilien-Betrachter - Immobilien-Informationen einsehen", - "fr": "Visualiseur immobilier - Consulter les informations immobilières" - }, - "accessRules": [ - # UI access to map view (read-only) - {"context": "UI", "item": "ui.feature.realestate.dashboard", "view": True}, - # Read-only DATA access (my records) - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, - ] + ], }, ] diff --git a/modules/features/teamsbot/datamodelTeamsbot.py b/modules/features/teamsbot/datamodelTeamsbot.py index bc17642f..f19b4c6c 100644 --- a/modules/features/teamsbot/datamodelTeamsbot.py +++ b/modules/features/teamsbot/datamodelTeamsbot.py @@ -9,6 +9,8 @@ from pydantic import BaseModel, Field from enum import Enum import uuid +from modules.datamodels.datamodelBase import PowerOnModel + # ============================================================================ # Enums @@ -72,7 +74,7 @@ class TeamsbotTransferMode(str, Enum): # Database Models (stored in PostgreSQL) # ============================================================================ -class TeamsbotSession(BaseModel): +class TeamsbotSession(PowerOnModel): """A Teams Bot meeting session.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Session ID") instanceId: str = Field(description="Feature instance ID (FK)") @@ -90,11 +92,9 @@ class TeamsbotSession(BaseModel): errorMessage: Optional[str] = Field(default=None, description="Error message if status is ERROR") transcriptSegmentCount: int = Field(default=0, description="Number of transcript segments in this session") botResponseCount: int = Field(default=0, description="Number of bot responses in this session") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of record creation") - lastModified: Optional[str] = Field(default=None, description="ISO timestamp of last modification") -class TeamsbotTranscript(BaseModel): +class TeamsbotTranscript(PowerOnModel): """A single transcript segment from the meeting.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Transcript segment ID") sessionId: str = Field(description="Session ID (FK)") @@ -105,10 +105,9 @@ class TeamsbotTranscript(BaseModel): language: Optional[str] = Field(default=None, description="Detected language code (e.g., de-DE)") isFinal: bool = Field(default=True, description="Whether this is a final or interim result") source: Optional[str] = Field(default=None, description="Source: caption, audioCapture, chat, chatHistory, speakerHint") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of record creation") -class TeamsbotBotResponse(BaseModel): +class TeamsbotBotResponse(PowerOnModel): """A bot response generated during a meeting session.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Response ID") sessionId: str = Field(description="Session ID (FK)") @@ -121,14 +120,13 @@ class TeamsbotBotResponse(BaseModel): processingTime: float = Field(default=0.0, description="Processing time in seconds") priceCHF: float = Field(default=0.0, description="Cost of this AI call in CHF") timestamp: Optional[str] = Field(default=None, description="ISO timestamp of the response") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of record creation") # ============================================================================ # System Bot Accounts (stored in PostgreSQL, credentials encrypted) # ============================================================================ -class TeamsbotSystemBot(BaseModel): +class TeamsbotSystemBot(PowerOnModel): """A system bot account for authenticated meeting joins. Credentials are stored encrypted in the database, NOT in the UI-visible config. Only mandate admins can manage system bots.""" @@ -138,15 +136,13 @@ class TeamsbotSystemBot(BaseModel): email: str = Field(description="Microsoft account email") encryptedPassword: str = Field(description="Encrypted Microsoft account password") isActive: bool = Field(default=True, description="Whether this bot account is active") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of creation") - lastModified: Optional[str] = Field(default=None, description="ISO timestamp of last modification") # ============================================================================ # User Account Credentials (stored in PostgreSQL, credentials encrypted) # ============================================================================ -class TeamsbotUserAccount(BaseModel): +class TeamsbotUserAccount(PowerOnModel): """Saved Microsoft credentials for 'Mein Account' joins. Each user can store their own MS credentials per mandate. Password is encrypted; on login only MFA confirmation is needed.""" @@ -156,15 +152,13 @@ class TeamsbotUserAccount(BaseModel): email: str = Field(description="Microsoft account email") encryptedPassword: str = Field(description="Encrypted Microsoft account password") displayName: Optional[str] = Field(default=None, description="Display name derived from MS account") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of creation") - lastModified: Optional[str] = Field(default=None, description="ISO timestamp of last modification") # ============================================================================ # Per-User Settings (stored in PostgreSQL, per user per instance) # ============================================================================ -class TeamsbotUserSettings(BaseModel): +class TeamsbotUserSettings(PowerOnModel): """Per-user settings for the Teams Bot feature. Each user has their own settings per feature instance. These override the instance-level defaults (TeamsbotConfig).""" @@ -182,8 +176,6 @@ class TeamsbotUserSettings(BaseModel): triggerCooldownSeconds: Optional[int] = Field(default=None, description="Trigger cooldown override") contextWindowSegments: Optional[int] = Field(default=None, description="Context window override") debugMode: Optional[bool] = Field(default=None, description="Debug mode override") - creationDate: Optional[str] = Field(default=None, description="ISO timestamp of creation") - lastModified: Optional[str] = Field(default=None, description="ISO timestamp of last modification") # ============================================================================ diff --git a/modules/features/teamsbot/interfaceFeatureTeamsbot.py b/modules/features/teamsbot/interfaceFeatureTeamsbot.py index 9be96393..4d6519d8 100644 --- a/modules/features/teamsbot/interfaceFeatureTeamsbot.py +++ b/modules/features/teamsbot/interfaceFeatureTeamsbot.py @@ -10,7 +10,6 @@ from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.shared.timeUtils import getIsoTimestamp from modules.shared.configuration import APP_CONFIG from .datamodelTeamsbot import ( @@ -104,13 +103,10 @@ class TeamsbotObjects: def createSession(self, sessionData: Dict[str, Any]) -> Dict[str, Any]: """Create a new session.""" - sessionData["creationDate"] = getIsoTimestamp() - sessionData["lastModified"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotSession, sessionData) def updateSession(self, sessionId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update session fields.""" - updates["lastModified"] = getIsoTimestamp() return self.db.recordModify(TeamsbotSession, sessionId, updates) def deleteSession(self, sessionId: str) -> bool: @@ -149,7 +145,6 @@ class TeamsbotObjects: def createTranscript(self, transcriptData: Dict[str, Any]) -> Dict[str, Any]: """Create a new transcript segment.""" - transcriptData["creationDate"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotTranscript, transcriptData) def updateTranscript(self, transcriptId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: @@ -180,7 +175,6 @@ class TeamsbotObjects: def createBotResponse(self, responseData: Dict[str, Any]) -> Dict[str, Any]: """Create a new bot response record.""" - responseData["creationDate"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotBotResponse, responseData) def _deleteResponsesBySession(self, sessionId: str) -> int: @@ -216,13 +210,10 @@ class TeamsbotObjects: def createSystemBot(self, botData: Dict[str, Any]) -> Dict[str, Any]: """Create a new system bot account.""" - botData["creationDate"] = getIsoTimestamp() - botData["lastModified"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotSystemBot, botData) def updateSystemBot(self, botId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update a system bot account.""" - updates["lastModified"] = getIsoTimestamp() return self.db.recordModify(TeamsbotSystemBot, botId, updates) def deleteSystemBot(self, botId: str) -> bool: @@ -243,13 +234,10 @@ class TeamsbotObjects: def createUserSettings(self, settingsData: Dict[str, Any]) -> Dict[str, Any]: """Create user settings.""" - settingsData["creationDate"] = getIsoTimestamp() - settingsData["lastModified"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotUserSettings, settingsData) def updateUserSettings(self, settingsId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update user settings.""" - updates["lastModified"] = getIsoTimestamp() return self.db.recordModify(TeamsbotUserSettings, settingsId, updates) def deleteUserSettings(self, settingsId: str) -> bool: @@ -270,13 +258,10 @@ class TeamsbotObjects: def createUserAccount(self, data: Dict[str, Any]) -> Dict[str, Any]: """Create saved MS credentials.""" - data["creationDate"] = getIsoTimestamp() - data["lastModified"] = getIsoTimestamp() return self.db.recordCreate(TeamsbotUserAccount, data) def updateUserAccount(self, accountId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Update saved MS credentials.""" - updates["lastModified"] = getIsoTimestamp() return self.db.recordModify(TeamsbotUserAccount, accountId, updates) def deleteUserAccount(self, accountId: str) -> bool: diff --git a/modules/features/teamsbot/mainTeamsbot.py b/modules/features/teamsbot/mainTeamsbot.py index 97cc107e..afdce822 100644 --- a/modules/features/teamsbot/mainTeamsbot.py +++ b/modules/features/teamsbot/mainTeamsbot.py @@ -103,25 +103,35 @@ TEMPLATE_ROLES = [ {"context": "RESOURCE", "item": "resource.feature.teamsbot.config.edit", "view": True}, ] }, + { + "roleLabel": "teamsbot-viewer", + "description": { + "en": "Teams Bot Viewer - View sessions and transcripts (read-only)", + "de": "Teams Bot Betrachter - Sitzungen und Transkripte ansehen (nur lesen)", + "fr": "Visualiseur Teams Bot - Consulter les sessions et transcriptions (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.teamsbot.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.teamsbot.sessions", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, { "roleLabel": "teamsbot-user", "description": { "en": "Teams Bot User - Can start/stop sessions and view transcripts", "de": "Teams Bot Benutzer - Kann Sitzungen starten/stoppen und Transkripte einsehen", - "fr": "Utilisateur Teams Bot - Peut démarrer/arrêter des sessions et voir les transcriptions" + "fr": "Utilisateur Teams Bot - Peut démarrer/arrêter des sessions et voir les transcriptions", }, "accessRules": [ - # UI access to dashboard and sessions (not settings) {"context": "UI", "item": "ui.feature.teamsbot.dashboard", "view": True}, {"context": "UI", "item": "ui.feature.teamsbot.sessions", "view": True}, - # Own records only {"context": "DATA", "item": "data.feature.teamsbot.TeamsbotSession", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, {"context": "DATA", "item": "data.feature.teamsbot.TeamsbotTranscript", "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, {"context": "DATA", "item": "data.feature.teamsbot.TeamsbotBotResponse", "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, - # Start and stop sessions {"context": "RESOURCE", "item": "resource.feature.teamsbot.session.start", "view": True}, {"context": "RESOURCE", "item": "resource.feature.teamsbot.session.stop", "view": True}, - ] + ], }, ] @@ -132,7 +142,7 @@ def getFeatureDefinition() -> Dict[str, Any]: "code": FEATURE_CODE, "label": FEATURE_LABEL, "icon": FEATURE_ICON, - "autoCreateInstance": True, + "autoCreateInstance": False, } diff --git a/modules/features/trustee/datamodelFeatureTrustee.py b/modules/features/trustee/datamodelFeatureTrustee.py index 538414a0..0889e361 100644 --- a/modules/features/trustee/datamodelFeatureTrustee.py +++ b/modules/features/trustee/datamodelFeatureTrustee.py @@ -5,11 +5,13 @@ from enum import Enum from typing import Optional from pydantic import BaseModel, Field + +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels import uuid -class TrusteeOrganisation(BaseModel): +class TrusteeOrganisation(PowerOnModel): """Represents trustee organisations (companies) within the Trustee feature.""" id: str = Field( # Unique string label (PK), not UUID description="Unique organisation identifier (label)", @@ -55,7 +57,7 @@ class TrusteeOrganisation(BaseModel): } ) # System attributes are automatically set by DatabaseConnector: - # _createdAt, _modifiedAt, _createdBy, _modifiedBy + # sysCreatedAt, sysModifiedAt, sysCreatedBy, sysModifiedBy (PowerOnModel) registerModelLabels( @@ -71,7 +73,7 @@ registerModelLabels( ) -class TrusteeRole(BaseModel): +class TrusteeRole(PowerOnModel): """Defines roles within the Trustee feature.""" id: str = Field( # Unique string label (PK), not UUID description="Unique role identifier (label)", @@ -122,7 +124,7 @@ registerModelLabels( ) -class TrusteeAccess(BaseModel): +class TrusteeAccess(PowerOnModel): """Defines user access to organisations with specific roles.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -207,7 +209,7 @@ registerModelLabels( ) -class TrusteeContract(BaseModel): +class TrusteeContract(PowerOnModel): """Defines customer contracts within organisations.""" id: str = Field( default_factory=lambda: str(uuid.uuid4()), @@ -289,7 +291,7 @@ class TrusteeDocumentTypeEnum(str, Enum): AUTO = "auto" -class TrusteeDocument(BaseModel): +class TrusteeDocument(PowerOnModel): """Contains document references for bookings. Documents reference files in the central Files table via fileId. @@ -413,7 +415,7 @@ registerModelLabels( ) -class TrusteePosition(BaseModel): +class TrusteePosition(PowerOnModel): """Contains booking positions (expense entries). A position can have up to two document references: documentId (Beleg) and bankDocumentId (Bank-Referenz). @@ -696,10 +698,6 @@ class TrusteePosition(BaseModel): } ) - # Allow extra fields like _createdAt from database - model_config = {"extra": "allow"} - - registerModelLabels( "TrusteePosition", {"en": "Position", "fr": "Position", "de": "Position"}, @@ -739,7 +737,7 @@ registerModelLabels( # ── TrusteeData* tables (synced from external accounting apps for analysis) ── -class TrusteeDataAccount(BaseModel): +class TrusteeDataAccount(PowerOnModel): """Chart of accounts synced from external accounting system.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) accountNumber: str = Field(description="Account number (e.g. '1020')") @@ -769,7 +767,7 @@ registerModelLabels( ) -class TrusteeDataJournalEntry(BaseModel): +class TrusteeDataJournalEntry(PowerOnModel): """Journal entry header synced from external accounting system.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) externalId: Optional[str] = Field(default=None, description="ID in the source system") @@ -799,7 +797,7 @@ registerModelLabels( ) -class TrusteeDataJournalLine(BaseModel): +class TrusteeDataJournalLine(PowerOnModel): """Journal entry line (debit/credit) synced from external accounting system.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) journalEntryId: str = Field(description="FK → TrusteeDataJournalEntry.id") @@ -833,7 +831,7 @@ registerModelLabels( ) -class TrusteeDataContact(BaseModel): +class TrusteeDataContact(PowerOnModel): """Customer or vendor synced from external accounting system.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) externalId: Optional[str] = Field(default=None, description="ID in the source system") @@ -873,7 +871,7 @@ registerModelLabels( ) -class TrusteeDataAccountBalance(BaseModel): +class TrusteeDataAccountBalance(PowerOnModel): """Account balance per period, derived from journal lines or directly from accounting system.""" id: str = Field(default_factory=lambda: str(uuid.uuid4())) accountNumber: str = Field(description="Account number") @@ -907,7 +905,7 @@ registerModelLabels( ) -class TrusteeAccountingConfig(BaseModel): +class TrusteeAccountingConfig(PowerOnModel): """Per-instance accounting system configuration with encrypted credentials. Each feature instance can connect to exactly one accounting system. @@ -946,7 +944,7 @@ registerModelLabels( ) -class TrusteeAccountingSync(BaseModel): +class TrusteeAccountingSync(PowerOnModel): """Tracks which position was synced to which external system and when. Used for duplicate prevention, audit trail, and retry logic. diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index b9a95005..7ed6fcff 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -1152,7 +1152,7 @@ class TrusteeObjects: logger.warning(f"Document {documentId} not found") return None - createdBy = existing.get("_createdBy") + createdBy = existing.get("sysCreatedBy") # Check system RBAC permission (userreport can only edit their own records) if not self.checkCombinedPermission(TrusteeDocument, "update", recordCreatedBy=createdBy): @@ -1178,7 +1178,7 @@ class TrusteeObjects: logger.warning(f"Document {documentId} not found") return False - createdBy = existing.get("_createdBy") + createdBy = existing.get("sysCreatedBy") if not self.checkCombinedPermission(TrusteeDocument, "delete", recordCreatedBy=createdBy): logger.warning(f"User {self.userId} lacks permission to delete document") @@ -1198,7 +1198,7 @@ class TrusteeObjects: def _toTrusteePositionOrDelete(self, rawRecord: Dict[str, Any], deleteCorrupt: bool = True) -> Optional[TrusteePosition]: """Build TrusteePosition safely; optionally delete irreparably corrupt records.""" - cleanRecord = {k: v for k, v in (rawRecord or {}).items() if not k.startswith("_") or k == "_createdAt"} + cleanRecord = {k: v for k, v in (rawRecord or {}).items() if not k.startswith("_") or k == "sysCreatedAt"} if not cleanRecord: return None @@ -1271,7 +1271,7 @@ class TrusteeObjects: """Get all positions with RBAC filtering and optional DB-level pagination. Filtering, sorting, and pagination are handled at the SQL level. - Post-processing cleans internal fields (keeps _createdAt) and validates + Post-processing cleans internal fields (keeps sysCreatedAt) and validates each record via _toTrusteePositionOrDelete (corrupt rows are deleted). NOTE(post-process): totalItems may slightly overcount when corrupt legacy @@ -1288,7 +1288,7 @@ class TrusteeObjects: featureCode=self.FEATURE_CODE ) - keepFields = {'_createdAt'} + keepFields = {'sysCreatedAt'} def _cleanAndValidate(records): items = [] @@ -1369,7 +1369,7 @@ class TrusteeObjects: logger.warning(f"Position {positionId} not found") return None - createdBy = existing.get("_createdBy") + createdBy = existing.get("sysCreatedBy") # Check system RBAC permission (userreport can only edit their own records) if not self.checkCombinedPermission(TrusteePosition, "update", recordCreatedBy=createdBy): @@ -1391,7 +1391,7 @@ class TrusteeObjects: logger.warning(f"Position {positionId} not found") return False - createdBy = existing.get("_createdBy") + createdBy = existing.get("sysCreatedBy") if not self.checkCombinedPermission(TrusteePosition, "delete", recordCreatedBy=createdBy): logger.warning(f"User {self.userId} lacks permission to delete position") diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 606da308..45824b1b 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -170,60 +170,81 @@ RESOURCE_OBJECTS = [ # Note: UI item=None means ALL views, specific items restrict to named views # IMPORTANT: item uses vollqualifizierte ObjectKeys (gemäss Navigation-API-Konzept) TEMPLATE_ROLES = [ + { + "roleLabel": "trustee-viewer", + "description": { + "en": "Trustee Viewer - View trustee data (read-only)", + "de": "Treuhand-Betrachter - Treuhand-Daten einsehen (nur lesen)", + "fr": "Visualiseur fiduciaire - Consulter les données fiduciaires (lecture seule)", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ], + }, + { + "roleLabel": "trustee-user", + "description": { + "en": "Trustee User - Create and manage own trustee records", + "de": "Treuhand-Benutzer - Eigene Treuhand-Daten erstellen und verwalten", + "fr": "Utilisateur fiduciaire - Créer et gérer ses propres données fiduciaires", + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, + {"context": "UI", "item": "ui.feature.trustee.expense-import", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, + ], + }, { "roleLabel": "trustee-admin", "description": { "en": "Trustee Administrator - Full access to all trustee data and settings", "de": "Treuhand-Administrator - Vollzugriff auf alle Treuhand-Daten und Einstellungen", - "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires" + "fr": "Administrateur fiduciaire - Accès complet aux données et paramètres fiduciaires", }, "accessRules": [ - # Full UI access (all views including admin views) {"context": "UI", "item": None, "view": True}, - # Full DATA access {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, - # Admin resource: manage instance roles {"context": "RESOURCE", "item": "resource.feature.trustee.instance-roles.manage", "view": True}, - ] + ], }, { "roleLabel": "trustee-accountant", "description": { "en": "Trustee Accountant - Manage accounting and financial data", "de": "Treuhand-Buchhalter - Buchhaltungs- und Finanzdaten verwalten", - "fr": "Comptable fiduciaire - Gérer les données comptables et financières" + "fr": "Comptable fiduciaire - Gérer les données comptables et financières", }, "accessRules": [ - # UI access to main views (not admin views, not expense-import) - vollqualifizierte ObjectKeys {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, {"context": "UI", "item": "ui.feature.trustee.settings", "view": True}, - # Group-level DATA access {"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"}, - # Accounting sync permission {"context": "RESOURCE", "item": "resource.feature.trustee.accounting.sync", "view": True}, {"context": "RESOURCE", "item": "resource.feature.trustee.accounting.view", "view": True}, - ] + ], }, { "roleLabel": "trustee-client", "description": { "en": "Trustee Client - View own accounting data and documents", "de": "Treuhand-Kunde - Eigene Buchhaltungsdaten und Dokumente einsehen", - "fr": "Client fiduciaire - Consulter ses propres données comptables et documents" + "fr": "Client fiduciaire - Consulter ses propres données comptables et documents", }, "accessRules": [ - # UI access to main views + expense-import - vollqualifizierte ObjectKeys {"context": "UI", "item": "ui.feature.trustee.dashboard", "view": True}, {"context": "UI", "item": "ui.feature.trustee.positions", "view": True}, {"context": "UI", "item": "ui.feature.trustee.documents", "view": True}, {"context": "UI", "item": "ui.feature.trustee.expense-import", "view": True}, {"context": "UI", "item": "ui.feature.trustee.scan-upload", "view": True}, - # Own records only (MY level) {"context": "DATA", "item": "data.feature.trustee.TrusteePosition", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, {"context": "DATA", "item": "data.feature.trustee.TrusteeDocument", "view": True, "read": "m", "create": "m", "update": "m", "delete": "n"}, - ] + ], }, ] diff --git a/modules/features/workspace/datamodelFeatureWorkspace.py b/modules/features/workspace/datamodelFeatureWorkspace.py index 80da5915..d7c292db 100644 --- a/modules/features/workspace/datamodelFeatureWorkspace.py +++ b/modules/features/workspace/datamodelFeatureWorkspace.py @@ -1,30 +1,15 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""Workspace feature data models — VoiceSettings and WorkspaceUserSettings.""" +"""Workspace feature data models — WorkspaceUserSettings.""" -from typing import Dict, Any, Optional +from typing import Optional from pydantic import BaseModel, Field +from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp import uuid -class VoiceSettings(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - userId: str = Field(description="ID of the user these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) - mandateId: str = Field(description="ID of the mandate these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) - featureInstanceId: str = Field(description="ID of the feature instance these settings belong to", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) - sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) - ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) - ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) - ttsVoiceMap: Dict[str, Any] = Field(default_factory=dict, description="Per-language voice mapping, e.g. {'de-DE': {'voiceName': 'de-DE-Wavenet-A'}, 'en-US': {'voiceName': 'en-US-Wavenet-C'}}", json_schema_extra={"frontend_type": "json", "frontend_readonly": False, "frontend_required": False}) - translationEnabled: bool = Field(default=True, description="Whether translation is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) - targetLanguage: str = Field(default="en-US", description="Target language for translation", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False}) - creationDate: float = Field(default_factory=getUtcTimestamp, description="Date when the settings were created (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) - lastModified: float = Field(default_factory=getUtcTimestamp, description="Date when the settings were last modified (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) - - -class WorkspaceUserSettings(BaseModel): +class WorkspaceUserSettings(PowerOnModel): """Per-user workspace settings. None values mean 'use instance default'.""" id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) userId: str = Field(description="User ID", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": True}) @@ -33,25 +18,6 @@ class WorkspaceUserSettings(BaseModel): maxAgentRounds: Optional[int] = Field(default=None, description="Max agent rounds override (None = instance default)", json_schema_extra={"frontend_type": "number", "frontend_readonly": False, "frontend_required": False}) -registerModelLabels( - "VoiceSettings", - {"en": "Voice Settings", "fr": "Paramètres vocaux"}, - { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance de fonctionnalité"}, - "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, - "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, - "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, - "ttsVoiceMap": {"en": "TTS Voice Map", "fr": "Carte des voix TTS"}, - "translationEnabled": {"en": "Translation Enabled", "fr": "Traduction activée"}, - "targetLanguage": {"en": "Target Language", "fr": "Langue cible"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, - "lastModified": {"en": "Last Modified", "fr": "Dernière modification"}, - }, -) - registerModelLabels( "WorkspaceUserSettings", {"en": "Workspace User Settings", "de": "Workspace Benutzereinstellungen"}, diff --git a/modules/features/workspace/interfaceFeatureWorkspace.py b/modules/features/workspace/interfaceFeatureWorkspace.py index bd1a03c4..05bda01d 100644 --- a/modules/features/workspace/interfaceFeatureWorkspace.py +++ b/modules/features/workspace/interfaceFeatureWorkspace.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Interface for Workspace feature — manages VoiceSettings and WorkspaceUserSettings. +Interface for Workspace feature — manages WorkspaceUserSettings. Uses a dedicated poweron_workspace database. """ @@ -10,11 +10,10 @@ from typing import Dict, Any, Optional from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.datamodels.datamodelUam import User -from modules.features.workspace.datamodelFeatureWorkspace import VoiceSettings, WorkspaceUserSettings +from modules.features.workspace.datamodelFeatureWorkspace import WorkspaceUserSettings from modules.interfaces.interfaceRbac import getRecordsetWithRBAC from modules.security.rbac import RbacClass from modules.shared.configuration import APP_CONFIG -from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) @@ -62,122 +61,6 @@ class WorkspaceObjects: self.featureInstanceId = featureInstanceId self.db.updateContext(self.userId) - # ========================================================================= - # VoiceSettings CRUD - # ========================================================================= - - def getVoiceSettings(self, userId: Optional[str] = None) -> Optional[VoiceSettings]: - try: - targetUserId = userId or self.userId - if not targetUserId: - logger.error("No user ID provided for voice settings") - return None - - recordFilter: Dict[str, Any] = {"userId": targetUserId} - if self.featureInstanceId: - recordFilter["featureInstanceId"] = self.featureInstanceId - - filteredSettings = getRecordsetWithRBAC( - self.db, VoiceSettings, self.currentUser, - recordFilter=recordFilter, mandateId=self.mandateId, - ) - - if not filteredSettings: - return None - - settingsData = filteredSettings[0] - if not settingsData.get("creationDate"): - settingsData["creationDate"] = getUtcTimestamp() - if not settingsData.get("lastModified"): - settingsData["lastModified"] = getUtcTimestamp() - - return VoiceSettings(**settingsData) - - except Exception as e: - logger.error(f"Error getting voice settings: {e}") - return None - - def createVoiceSettings(self, settingsData: Dict[str, Any]) -> Dict[str, Any]: - try: - if "userId" not in settingsData: - settingsData["userId"] = self.userId - if "mandateId" not in settingsData: - settingsData["mandateId"] = self.mandateId - if "featureInstanceId" not in settingsData: - settingsData["featureInstanceId"] = self.featureInstanceId - - existing = self.getVoiceSettings(settingsData["userId"]) - if existing: - raise ValueError(f"Voice settings already exist for user {settingsData['userId']}") - - createdRecord = self.db.recordCreate(VoiceSettings, settingsData) - if not createdRecord or not createdRecord.get("id"): - raise ValueError("Failed to create voice settings record") - - logger.info(f"Created voice settings for user {settingsData['userId']}") - return createdRecord - - except Exception as e: - logger.error(f"Error creating voice settings: {e}") - raise - - def updateVoiceSettings(self, userId: str, updateData: Dict[str, Any]) -> Dict[str, Any]: - try: - existing = self.getVoiceSettings(userId) - if not existing: - raise ValueError(f"Voice settings not found for user {userId}") - - updateData["lastModified"] = getUtcTimestamp() - success = self.db.recordModify(VoiceSettings, existing.id, updateData) - if not success: - raise ValueError("Failed to update voice settings record") - - updated = self.getVoiceSettings(userId) - if not updated: - raise ValueError("Failed to retrieve updated voice settings") - - logger.info(f"Updated voice settings for user {userId}") - return updated.model_dump() - - except Exception as e: - logger.error(f"Error updating voice settings: {e}") - raise - - def deleteVoiceSettings(self, userId: str) -> bool: - try: - existing = self.getVoiceSettings(userId) - if not existing: - return False - success = self.db.recordDelete(VoiceSettings, existing.id) - if success: - logger.info(f"Deleted voice settings for user {userId}") - return success - except Exception as e: - logger.error(f"Error deleting voice settings: {e}") - return False - - def getOrCreateVoiceSettings(self, userId: Optional[str] = None) -> VoiceSettings: - targetUserId = userId or self.userId - if not targetUserId: - raise ValueError("No user ID provided for voice settings") - - existing = self.getVoiceSettings(targetUserId) - if existing: - return existing - - defaultSettings = { - "userId": targetUserId, - "mandateId": self.mandateId, - "featureInstanceId": self.featureInstanceId, - "sttLanguage": "de-DE", - "ttsLanguage": "de-DE", - "ttsVoice": "de-DE-KatjaNeural", - "translationEnabled": True, - "targetLanguage": "en-US", - } - createdRecord = self.createVoiceSettings(defaultSettings) - return VoiceSettings(**createdRecord) - # ========================================================================= # WorkspaceUserSettings CRUD # ========================================================================= diff --git a/modules/features/workspace/mainWorkspace.py b/modules/features/workspace/mainWorkspace.py index c502a82e..5ef9b399 100644 --- a/modules/features/workspace/mainWorkspace.py +++ b/modules/features/workspace/mainWorkspace.py @@ -128,7 +128,7 @@ TEMPLATE_ROLES = [ "accessRules": [ {"context": "UI", "item": None, "view": True}, {"context": "RESOURCE", "item": None, "view": True}, - # DATA: never ALL in shared instances — every role (including admin) sees only _createdBy = self + # DATA: never ALL in shared instances — every role (including admin) sees only sysCreatedBy = self {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, ] }, diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index 6a3500ce..7feef4db 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -76,6 +76,27 @@ class _PendingEditsStore: _pendingEditsStore = _PendingEditsStore() +def _workspaceBillingFeatureCode(user, mandateId: Optional[str], instanceId: str) -> Optional[str]: + """Resolve FeatureInstance.featureCode for billing/UI when workflow is not on ServiceCenterContext.""" + if not instanceId or not str(instanceId).strip(): + return None + try: + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + + appIf = getAppInterface(user, mandateId=mandateId or None) + inst = appIf.getFeatureInstance(str(instanceId).strip()) + if not inst: + return None + if isinstance(inst, dict): + code = inst.get("featureCode") + else: + code = getattr(inst, "featureCode", None) + return str(code).strip() if code else None + except Exception as e: + logger.debug("Workspace: feature code lookup failed for instance %s: %s", instanceId, e) + return None + + class WorkspaceInputRequest(BaseModel): """Prompt input for the unified workspace.""" prompt: str = Field(description="User prompt text") @@ -87,6 +108,7 @@ class WorkspaceInputRequest(BaseModel): workflowId: Optional[str] = Field(default=None, description="Continue existing workflow") userLanguage: str = Field(default="en", description="User language code") allowedProviders: List[str] = Field(default_factory=list, description="Restrict AI to these providers") + requireNeutralization: Optional[bool] = Field(default=None, description="Per-request neutralization override") async def _getAiObjects() -> AiObjects: @@ -546,11 +568,13 @@ async def streamWorkspaceStart( from modules.serviceCenter import getService from modules.serviceCenter.context import ServiceCenterContext + wsBillingFeatureCode = _workspaceBillingFeatureCode(context.user, mandateId or "", instanceId) svcCtx = ServiceCenterContext( user=context.user, mandate_id=mandateId or "", feature_instance_id=instanceId, workflow_id=workflowId, + feature_code=wsBillingFeatureCode, ) chatSvc = getService("chat", svcCtx) attachmentLabel = _buildWorkspaceAttachmentLabel( @@ -589,6 +613,8 @@ async def streamWorkspaceStart( userLanguage=userInput.userLanguage, instanceConfig=instanceConfig, allowedProviders=userInput.allowedProviders, + requireNeutralization=userInput.requireNeutralization, + billingFeatureCode=wsBillingFeatureCode, ) ) eventManager.register_agent_task(queueId, agentTask) @@ -644,6 +670,8 @@ async def _runWorkspaceAgent( userLanguage: str = "en", instanceConfig: Dict[str, Any] = None, allowedProviders: List[str] = None, + requireNeutralization: Optional[bool] = None, + billingFeatureCode: Optional[str] = None, ): """Run the serviceAgent loop and forward events to the SSE queue.""" try: @@ -654,6 +682,7 @@ async def _runWorkspaceAgent( mandate_id=mandateId, feature_instance_id=instanceId, workflow_id=workflowId, + feature_code=billingFeatureCode, ) agentService = getService("agent", ctx) chatService = getService("chat", ctx) @@ -661,6 +690,11 @@ async def _runWorkspaceAgent( if allowedProviders: aiService.services.allowedProviders = allowedProviders + logger.info(f"Workspace agent: allowedProviders={allowedProviders}") + else: + logger.debug("Workspace agent: no allowedProviders in request") + if requireNeutralization is not None: + ctx.requireNeutralization = requireNeutralization wfRecord = chatInterface.getWorkflow(workflowId) if workflowId else None wfName = "" @@ -888,12 +922,30 @@ async def listWorkspaceWorkflows( request: Request, instanceId: str = Path(...), includeArchived: bool = Query(default=False, description="Include archived workflows"), + search: str = Query(default="", description="Fulltext search in workflow titles and message content"), context: RequestContext = Depends(getRequestContext), ): """List workspace workflows/conversations for this instance.""" _validateInstanceAccess(instanceId, context) chatInterface = _getChatInterface(context, featureInstanceId=instanceId) workflows = chatInterface.getWorkflows() or [] + + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + _fiCache: Dict[str, Dict[str, str]] = {} + + def _resolveFeatureLabels(fiId: str) -> Dict[str, str]: + if fiId not in _fiCache: + fi = rootIf.getFeatureInstance(fiId) + if fi: + _fiCache[fiId] = { + "featureLabel": getattr(fi, "label", "") or getattr(fi, "featureCode", fiId), + "featureCode": getattr(fi, "featureCode", ""), + } + else: + _fiCache[fiId] = {"featureLabel": fiId[:8], "featureCode": ""} + return _fiCache[fiId] + items = [] for wf in workflows: if isinstance(wf, dict): @@ -905,13 +957,63 @@ async def listWorkspaceWorkflows( "status": getattr(wf, "status", ""), "startedAt": getattr(wf, "startedAt", None), "lastActivity": getattr(wf, "lastActivity", None), + "featureInstanceId": getattr(wf, "featureInstanceId", instanceId), } if not includeArchived and item.get("status") == "archived": continue + fiId = item.get("featureInstanceId") or instanceId + labels = _resolveFeatureLabels(fiId) + item.setdefault("featureLabel", labels["featureLabel"]) + item.setdefault("featureCode", labels["featureCode"]) + item.setdefault("featureInstanceId", fiId) + + lastMsg = chatInterface.getLastMessageTimestamp(item.get("id")) + if lastMsg: + item["lastMessageAt"] = lastMsg + items.append(item) + + if search and search.strip(): + searchLower = search.strip().lower() + matchedIds = set() + for item in items: + if searchLower in (item.get("name") or "").lower() or searchLower in (item.get("label") or "").lower(): + matchedIds.add(item["id"]) + contentHits = chatInterface.searchWorkflowsByContent(searchLower, limit=50) + matchedIds.update(contentHits) + items = [i for i in items if i["id"] in matchedIds] + return JSONResponse({"workflows": items}) +class ResolveRagRequest(BaseModel): + """Request body for resolving a chat via RAG.""" + chatId: str = Field(..., description="Workflow/chat ID to resolve") + + +@router.post("/{instanceId}/resolve-rag") +@limiter.limit("60/minute") +async def resolveRag( + request: Request, + instanceId: str = Path(...), + body: ResolveRagRequest = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Build a RAG summary for a chat (workflow) to inject into the input area.""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + messages = chatInterface.getMessages(body.chatId) or [] + + texts = [] + for msg in messages[:30]: + content = msg.get("message") if isinstance(msg, dict) else getattr(msg, "message", "") + if content: + texts.append(content[:500]) + + summary = "\n---\n".join(texts[:10]) if texts else "" + return JSONResponse({"summary": summary, "chatId": body.chatId, "messageCount": len(texts)}) + + class UpdateWorkflowRequest(BaseModel): """Request body for updating a workflow (PATCH).""" name: Optional[str] = Field(default=None, description="New workflow name") @@ -1583,137 +1685,6 @@ async def synthesizeVoice( return JSONResponse({"audio": None, "note": "TTS via browser Speech Synthesis API recommended"}) -# ========================================================================= -# Voice Settings Endpoints -# ========================================================================= - -@router.get("/{instanceId}/settings/voice") -@limiter.limit("120/minute") -async def getVoiceSettings( - request: Request, - instanceId: str = Path(...), - context: RequestContext = Depends(getRequestContext), -): - """Load voice settings for the current user and instance.""" - _validateInstanceAccess(instanceId, context) - wsInterface = _getWorkspaceInterface(context, instanceId) - userId = str(context.user.id) - try: - vs = wsInterface.getVoiceSettings(userId) - if not vs: - logger.info(f"GET voice settings: not found for user={userId}, creating defaults") - vs = wsInterface.getOrCreateVoiceSettings(userId) - result = vs.model_dump() if vs else {} - mapKeys = list(result.get("ttsVoiceMap", {}).keys()) if result else [] - logger.info(f"GET voice settings for user={userId}: ttsVoiceMap languages={mapKeys}") - return JSONResponse(result) - except Exception as e: - logger.error(f"Failed to load voice settings for user={userId}: {e}", exc_info=True) - return JSONResponse({"ttsVoiceMap": {}}, status_code=200) - - -@router.put("/{instanceId}/settings/voice") -@limiter.limit("120/minute") -async def updateVoiceSettings( - request: Request, - instanceId: str = Path(...), - body: dict = Body(...), - context: RequestContext = Depends(getRequestContext), -): - """Update voice settings for the current user and instance.""" - _validateInstanceAccess(instanceId, context) - wsInterface = _getWorkspaceInterface(context, instanceId) - userId = str(context.user.id) - - try: - logger.info(f"PUT voice settings for user={userId}, instance={instanceId}, body keys={list(body.keys())}") - vs = wsInterface.getVoiceSettings(userId) - if not vs: - logger.info(f"No existing voice settings, creating new for user={userId}") - createData = { - "userId": userId, - "mandateId": str(context.mandateId) if context.mandateId else "", - "featureInstanceId": instanceId, - } - createData.update(body) - created = wsInterface.createVoiceSettings(createData) - logger.info(f"Created voice settings for user={userId}, ttsVoiceMap keys={list((created or {}).get('ttsVoiceMap', {}).keys())}") - return JSONResponse(created) - - updateData = {k: v for k, v in body.items() if k not in ("id", "userId", "mandateId", "featureInstanceId", "creationDate")} - logger.info(f"Updating voice settings for user={userId}, update keys={list(updateData.keys())}") - updated = wsInterface.updateVoiceSettings(userId, updateData) - logger.info(f"Updated voice settings for user={userId}, ttsVoiceMap keys={list((updated or {}).get('ttsVoiceMap', {}).keys())}") - return JSONResponse(updated) - except Exception as e: - logger.error(f"Failed to update voice settings for user={userId}: {e}", exc_info=True) - return JSONResponse({"error": str(e)}, status_code=500) - - -@router.get("/{instanceId}/voice/languages") -@limiter.limit("120/minute") -async def getVoiceLanguages( - request: Request, - instanceId: str = Path(...), - context: RequestContext = Depends(getRequestContext), -): - """Return available TTS languages.""" - mandateId, _ = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - languagesResult = await voiceInterface.getAvailableLanguages() - languageList = languagesResult.get("languages", []) if isinstance(languagesResult, dict) else languagesResult - return JSONResponse({"languages": languageList}) - - -@router.get("/{instanceId}/voice/voices") -@limiter.limit("120/minute") -async def getVoiceVoices( - request: Request, - instanceId: str = Path(...), - language: str = Query("de-DE"), - context: RequestContext = Depends(getRequestContext), -): - """Return available TTS voices for a given language.""" - mandateId, _ = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - voicesResult = await voiceInterface.getAvailableVoices(language) - voiceList = voicesResult.get("voices", []) if isinstance(voicesResult, dict) else voicesResult - return JSONResponse({"voices": voiceList}) - - -@router.post("/{instanceId}/voice/test") -@limiter.limit("30/minute") -async def testVoice( - request: Request, - instanceId: str = Path(...), - body: dict = Body(...), - context: RequestContext = Depends(getRequestContext), -): - """Test a specific voice with a sample text.""" - import base64 - mandateId, _ = _validateInstanceAccess(instanceId, context) - text = body.get("text", "Hallo, das ist ein Stimmtest.") - language = body.get("language", "de-DE") - voiceId = body.get("voiceId") - - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - - try: - result = await voiceInterface.textToSpeech(text=text, languageCode=language, voiceName=voiceId) - if result and isinstance(result, dict): - audioContent = result.get("audioContent") - if audioContent: - audioB64 = base64.b64encode( - audioContent if isinstance(audioContent, bytes) else audioContent.encode() - ).decode() - return JSONResponse({"success": True, "audio": audioB64, "format": "mp3", "text": text}) - return JSONResponse({"success": False, "error": "TTS returned no audio"}) - except Exception as e: - logger.error(f"Voice test failed: {e}") - raise HTTPException(status_code=500, detail=f"TTS test failed: {str(e)}") # ============================================================================= diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 89cf4126..93b17d6a 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -11,9 +11,9 @@ Multi-Tenant Design: """ import logging -from typing import Optional, Dict +from typing import Optional, Dict, Tuple from passlib.context import CryptContext -from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.connectors.connectorDbPostgre import DatabaseConnector, _get_cached_connector from modules.shared.configuration import APP_CONFIG from modules.datamodels.datamodelUam import ( Mandate, @@ -38,6 +38,89 @@ pwdContext = CryptContext(schemes=["argon2"], deprecated="auto") # Cache für Role-IDs (roleLabel -> roleId) _roleIdCache: Dict[str, str] = {} +# PowerOn logical databases to scan (same set as gateway/scripts/script_db_export_migration.py ALL_DATABASES). +_POWERON_DATABASE_NAMES: Tuple[str, ...] = ( + "poweron_app", + "poweron_automation", + "poweron_automation2", + "poweron_billing", + "poweron_chat", + "poweron_chatbot", + "poweron_commcoach", + "poweron_knowledge", + "poweron_management", + "poweron_neutralization", + "poweron_realestate", + "poweron_teamsbot", + "poweron_test", + "poweron_trustee", + "poweron_workspace", +) + + +def _configPrefixForPoweronDatabase(dbName: str) -> str: + return { + "poweron_app": "DB_APP", + "poweron_chat": "DB_CHAT", + "poweron_chatbot": "DB_CHATBOT", + "poweron_management": "DB_MANAGEMENT", + "poweron_realestate": "DB_REALESTATE", + "poweron_trustee": "DB_TRUSTEE", + # Same as initAutomationTemplates: default DB_* (not a separate DB_AUTOMATION_* prefix). + "poweron_automation": "DB", + "poweron_billing": "DB", + }.get(dbName, "DB") + + +def _openConnectorForPoweronDatabase(dbName: str) -> Optional[DatabaseConnector]: + """Connect to a named PowerOn database using DB_* / DB_APP_* style config (shared with export script).""" + prefix = _configPrefixForPoweronDatabase(dbName) + host = APP_CONFIG.get(f"{prefix}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") + user = APP_CONFIG.get(f"{prefix}_USER") or APP_CONFIG.get("DB_USER") + password = APP_CONFIG.get(f"{prefix}_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD_SECRET") + portRaw = APP_CONFIG.get(f"{prefix}_PORT") or APP_CONFIG.get("DB_PORT", 5432) + try: + port = int(portRaw) + except (TypeError, ValueError): + port = 5432 + if not user or not password: + logger.debug( + f"bootstrap: skip legacy _* -> sys* migration for {dbName} (missing credentials for {prefix})" + ) + return None + try: + return _get_cached_connector( + dbHost=host, + dbDatabase=dbName, + dbUser=user, + dbPassword=password, + dbPort=port, + userId=None, + ) + except Exception as e: + logger.warning(f"bootstrap: cannot open {dbName} for legacy _* -> sys* migration: {e}") + return None + + +def migrateLegacyUnderscoreSysColumnsAllPoweronDatabases() -> None: + """ + Run DatabaseConnector.migrateLegacyUnderscoreSysColumns on every configured PowerOn database. + Actual table scan and SQL live in the connector module. + """ + grandTotal = 0 + for dbName in _POWERON_DATABASE_NAMES: + conn = _openConnectorForPoweronDatabase(dbName) + if not conn: + continue + try: + grandTotal += conn.migrateLegacyUnderscoreSysColumns() + except Exception as e: + logger.warning(f"bootstrap: migrateLegacyUnderscoreSysColumns failed for {dbName}: {e}") + if grandTotal: + logger.info( + f"bootstrap: legacy _* -> sys* migration total {grandTotal} cell(s) across PowerOn databases" + ) + def initBootstrap(db: DatabaseConnector) -> None: """ @@ -50,6 +133,9 @@ def initBootstrap(db: DatabaseConnector) -> None: # Initialize root mandate mandateId = initRootMandate(db) + + # Copy legacy _createdAt/_createdBy/_modifiedAt/_modifiedBy into sys* on all PowerOn DBs (connector routine) + migrateLegacyUnderscoreSysColumnsAllPoweronDatabases() # Migrate existing mandate records: description -> label _migrateMandateDescriptionToLabel(db) @@ -92,8 +178,38 @@ def initBootstrap(db: DatabaseConnector) -> None: # Seed automation templates (after admin user exists) initAutomationTemplates(db, adminUserId) - # Initialize feature instances for root mandate - if mandateId: + # Run root-user migration (one-time, sets completion flag) + migrationDone = False + try: + from modules.migration.migrateRootUsers import migrateRootUsers, _isMigrationCompleted + migrationDone = _isMigrationCompleted(db) + if not migrationDone: + # Create root instances first (needed for migration), then migrate + if mandateId: + initRootMandateFeatures(db, mandateId) + result = migrateRootUsers(db) + migrationDone = result.get("status") != "error" + else: + migrationDone = True + except Exception as e: + logger.error(f"Root user migration failed: {e}") + + # Run voice & documents migration (one-time, sets completion flag) + try: + from modules.migration.migrateVoiceAndDocuments import migrateVoiceAndDocuments + migrateVoiceAndDocuments(db) + except Exception as e: + logger.error(f"Voice & documents migration failed: {e}") + + # Backfill FileContentIndex scope fields from FileItem (one-time) + try: + from modules.migration.migrateRagScopeFields import runMigration as migrateRagScope + migrateRagScope(appDb=db) + except Exception as e: + logger.error(f"RAG scope fields migration failed: {e}") + + # After migration: root mandate is purely technical — no feature instances + if not migrationDone and mandateId: initRootMandateFeatures(db, mandateId) # Remove feature instances for features that no longer exist in the codebase @@ -110,18 +226,26 @@ def initBootstrap(db: DatabaseConnector) -> None: # Auto-provision Stripe Products/Prices for paid plans (idempotent) _bootstrapStripePrices() + # Purge soft-deleted mandates past 30-day retention + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rootIf.purgeExpiredMandates(retentionDays=30) + except Exception as e: + logger.warning(f"Mandate retention purge failed: {e}") + def initAutomationTemplates(dbApp: DatabaseConnector, adminUserId: Optional[str] = None) -> None: """ Seed initial automation templates from subAutomationTemplates.py. Only runs if no templates exist yet (bootstrap). - Creates templates with _createdBy = admin user (SysAdmin privilege). + Creates templates with sysCreatedBy = admin user (SysAdmin privilege). NOTE: AutomationTemplate lives in poweron_automation database, not poweron_app! Args: dbApp: Database connector for poweron_app (used to get admin user if needed) - adminUserId: Admin user ID for _createdBy field + adminUserId: Admin user ID for sysCreatedBy field """ import json from modules.features.automation.subAutomationTemplates import AUTOMATION_TEMPLATES @@ -2004,71 +2128,43 @@ def _createStoreResourceRules(db: DatabaseConnector) -> None: def initRootMandateBilling(mandateId: str) -> None: """ - Initialize billing settings for root mandate. - Root mandate uses PREPAY_USER model with default initial credit per user in settings (DEFAULT_USER_CREDIT_CHF at bootstrap only). - Creates billing accounts for ALL users regardless of billing model (for audit trail). - - Args: - mandateId: Root mandate ID + Initialize billing settings for root mandate (PREPAY_MANDATE). + Creates mandate pool account and user audit accounts. """ try: from modules.interfaces.interfaceDbBilling import _getRootInterface from modules.interfaces.interfaceDbApp import getRootInterface as getAppRootInterface - from modules.datamodels.datamodelBilling import ( - BillingSettings, - BillingModelEnum, - DEFAULT_USER_CREDIT_CHF, - parseBillingModelFromStoredValue, - ) + from modules.datamodels.datamodelBilling import BillingSettings billingInterface = _getRootInterface() appInterface = getAppRootInterface() - # Check if settings already exist existingSettings = billingInterface.getSettings(mandateId) if existingSettings: logger.info("Billing settings for root mandate already exist") else: settings = BillingSettings( mandateId=mandateId, - billingModel=BillingModelEnum.PREPAY_USER, - defaultUserCredit=DEFAULT_USER_CREDIT_CHF, warningThresholdPercent=10.0, notifyOnWarning=True ) - billingInterface.createSettings(settings) - logger.info( - f"Created billing settings for root mandate: PREPAY_USER with {DEFAULT_USER_CREDIT_CHF} CHF default credit" - ) + logger.info("Created billing settings for root mandate: PREPAY_MANDATE") existingSettings = billingInterface.getSettings(mandateId) - # Always create user accounts for all users (audit trail) if existingSettings: - billingModel = parseBillingModelFromStoredValue( - existingSettings.get("billingModel") - ).value - - # Initial balance depends on billing model - if billingModel == BillingModelEnum.PREPAY_USER.value: - initialBalance = float(existingSettings.get("defaultUserCredit", 0.0)) - else: - initialBalance = 0.0 # PREPAY_MANDATE: budget on pool account - + billingInterface.getOrCreateMandateAccount(mandateId, initialBalance=0.0) userMandates = appInterface.getUserMandatesByMandate(mandateId) accountsCreated = 0 - for um in userMandates: userId = um.get("userId") if isinstance(um, dict) else getattr(um, "userId", None) if userId: existingAccount = billingInterface.getUserAccount(mandateId, userId) if not existingAccount: - billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=initialBalance) + billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=0.0) accountsCreated += 1 - logger.debug(f"Created billing account for user {userId}") - if accountsCreated > 0: - logger.info(f"Created {accountsCreated} billing accounts for root mandate users with {initialBalance} CHF each") + logger.info(f"Created {accountsCreated} billing audit accounts for root mandate users") except Exception as e: logger.warning(f"Failed to initialize root mandate billing (non-critical): {e}") diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 12eb935b..d980eb56 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -187,12 +187,8 @@ class AppObjects: # Complex objects that should be filtered out objectFields[fieldName] = value else: - # Field not in model - treat as scalar if simple, otherwise filter out - # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector - if fieldName.startswith("_"): - # Metadata fields should be passed through to connector - simpleFields[fieldName] = value - elif isinstance(value, (str, int, float, bool, type(None))): + # Field not in model - pass through scalars; nested objects go to objectFields + if isinstance(value, (str, int, float, bool, type(None))): simpleFields[fieldName] = value else: objectFields[fieldName] = value @@ -528,7 +524,7 @@ class AppObjects: items = [] for record in result["items"]: - cleanedUser = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedUser = dict(record) if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] items.append(User(**cleanedUser)) @@ -560,7 +556,7 @@ class AppObjects: # Return first matching user (should be unique) userDict = users[0] # Filter out database-specific fields - cleanedUser = {k: v for k, v in userDict.items() if not k.startswith("_")} + cleanedUser = dict(userDict) # Ensure roleLabels is always a list, not None if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] @@ -586,7 +582,7 @@ class AppObjects: # User already filtered by RBAC, just clean fields user_dict = users[0] - cleanedUser = {k: v for k, v in user_dict.items() if not k.startswith("_")} + cleanedUser = dict(user_dict) # Ensure roleLabels is always a list, not None if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] @@ -648,12 +644,10 @@ class AppObjects: if not self._verifyPassword(password, userRecord["hashedPassword"]): raise ValueError("Invalid password") - # Return clean User object (without password hash and internal fields) - cleanedUser = {k: v for k, v in userRecord.items() if not k.startswith("_") and k != "hashedPassword" and k != "resetToken" and k != "resetTokenExpires"} - # Ensure roleLabels is always a list - if cleanedUser.get("roleLabels") is None: - cleanedUser["roleLabels"] = [] - return User(**cleanedUser) + user = User.model_validate(userRecord) + if user.roleLabels is None: + return user.model_copy(update={"roleLabels": []}) + return user def createUser( self, @@ -734,9 +728,8 @@ class AppObjects: # Clear cache to ensure fresh data (already done above) - # Assign new user to the root mandate with mandate-instance 'user' role (no feature instances) - userId = createdUser[0]["id"] - self._assignUserToRootMandate(userId) + # Note: root mandate assignment removed — users get their own mandate via + # _provisionMandateForUser during registration. Root mandate is purely technical. return User(**createdUser[0]) @@ -801,48 +794,6 @@ class AppObjects: logger.error(f"Error updating user: {str(e)}") raise ValueError(f"Failed to update user: {str(e)}") - def _assignUserToRootMandate(self, userId: str) -> None: - """ - Assign a new user to the root mandate with the mandate-instance 'user' role. - This ensures every user has a base membership in the system mandate. - - Uses the mandate-instance role (mandateId=rootMandateId), not the global template. - Feature instance access is NOT granted here - it is managed separately - via invitations or admin assignment. - - Args: - userId: User ID to assign - """ - try: - from modules.datamodels.datamodelRbac import Role - - rootMandateId = self._getRootMandateId() - if not rootMandateId: - logger.warning("No root mandate found, skipping root mandate assignment") - return - - # Check if user already has a mandate membership - existing = self.getUserMandate(userId, rootMandateId) - if existing: - logger.debug(f"User {userId} already assigned to root mandate") - return - - # Mandate-instance 'user' role (bound to this mandate, not a global template) - mandateUserRoles = self.db.getRecordset( - Role, - recordFilter={"roleLabel": "user", "mandateId": rootMandateId, "featureInstanceId": None} - ) - userRoleId = mandateUserRoles[0].get("id") if mandateUserRoles else None - - roleIds = [userRoleId] if userRoleId else [] - - self.createUserMandate(userId, rootMandateId, roleIds) - logger.info(f"Assigned user {userId} to root mandate with user role") - - except Exception as e: - # Log but don't fail user creation - logger.error(f"Error assigning user {userId} to root mandate: {e}") - def disableUser(self, userId: str) -> User: """Disables a user if current user has permission.""" return self.updateUser(userId, {"enabled": False}) @@ -920,7 +871,7 @@ class AppObjects: result = [] for userRecord in users: - cleanedUser = {k: v for k, v in userRecord.items() if not k.startswith("_")} + cleanedUser = dict(userRecord) if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] result.append(User(**cleanedUser)) @@ -960,7 +911,7 @@ class AppObjects: ) if users: - cleanedUser = {k: v for k, v in users[0].items() if not k.startswith("_")} + cleanedUser = dict(users[0]) if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] return User(**cleanedUser) @@ -1021,7 +972,7 @@ class AppObjects: ) if users: - cleanedUser = {k: v for k, v in users[0].items() if not k.startswith("_")} + cleanedUser = dict(users[0]) if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] return User(**cleanedUser) @@ -1084,7 +1035,7 @@ class AppObjects: logger.warning(f"Reset token expired for user {userRecord.get('id')}") return None - cleanedUser = {k: v for k, v in userRecord.items() if not k.startswith("_")} + cleanedUser = dict(userRecord) if cleanedUser.get("roleLabels") is None: cleanedUser["roleLabels"] = [] return User(**cleanedUser) @@ -1372,7 +1323,7 @@ class AppObjects: # Filter out database-specific fields filteredMandates = [] for mandate in allMandates: - cleanedMandate = {k: v for k, v in mandate.items() if not k.startswith("_")} + cleanedMandate = dict(mandate) filteredMandates.append(cleanedMandate) # If no pagination requested, return all items @@ -1421,7 +1372,7 @@ class AppObjects: # Filter out database-specific fields filteredMandates = [] for mandate in mandates: - cleanedMandate = {k: v for k, v in mandate.items() if not k.startswith("_")} + cleanedMandate = dict(mandate) filteredMandates.append(cleanedMandate) if not filteredMandates: return None @@ -1456,6 +1407,171 @@ class AppObjects: return Mandate(**createdRecord) + def _provisionMandateForUser(self, userId: str, mandateName: str, planKey: str) -> Dict[str, Any]: + """ + Atomic provisioning: create Mandate + UserMandate + Subscription + auto-create FeatureInstances. + Internal method — bypasses RBAC (used during registration when user has no permissions yet). + """ + from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS + from modules.datamodels.datamodelFeatures import FeatureInstance + from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate + from modules.interfaces.interfaceFeatures import getFeatureInterface + from modules.system.registry import loadFeatureMainModules + + plan = BUILTIN_PLANS.get(planKey) + if not plan: + raise ValueError(f"Unknown plan: {planKey}") + + mandateData = Mandate( + name=mandateName, + label=mandateName, + enabled=True, + isSystem=False, + ) + createdMandate = self.db.recordCreate(Mandate, mandateData) + if not createdMandate or not createdMandate.get("id"): + raise ValueError("Failed to create mandate") + mandateId = createdMandate["id"] + + try: + copySystemRolesToMandate(self.db, mandateId) + + adminRoleId = None + mandateRoles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId, "featureInstanceId": None}) + for r in mandateRoles: + if "admin" in (r.get("roleLabel") or "").lower(): + adminRoleId = r.get("id") + break + + if not adminRoleId: + raise ValueError(f"No admin role found for mandate {mandateId} — cannot assign user without role") + + self.createUserMandate(userId, mandateId, roleIds=[adminRoleId], skipCapacityCheck=True) + + subscription = MandateSubscription( + mandateId=mandateId, + planKey=planKey, + status=SubscriptionStatusEnum.PENDING, + ) + if plan.trialDays: + pass # trialEndsAt set on ACTIVE/TRIALING transition + from modules.interfaces.interfaceDbSubscription import _getRootInterface as _getSubRoot + subInterface = _getSubRoot() + subInterface.createSubscription(subscription) + + featureInterface = getFeatureInterface(self.db) + mainModules = loadFeatureMainModules() + createdInstances = [] + for featureName, module in mainModules.items(): + if not hasattr(module, "getFeatureDefinition"): + continue + try: + featureDef = module.getFeatureDefinition() + if not featureDef.get("autoCreateInstance", False): + continue + featureCode = featureDef.get("code", featureName) + featureLabel = featureDef.get("label", {}).get("en", featureName) + instance = featureInterface.createFeatureInstance( + featureCode=featureCode, + mandateId=mandateId, + label=featureLabel, + enabled=True, + copyTemplateRoles=True, + ) + if instance: + instanceId = instance.get("id") if isinstance(instance, dict) else instance.id + createdInstances.append(instanceId) + instanceRoles = self.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) + adminInstRoleId = None + for ir in instanceRoles: + roleLabel = (ir.get("roleLabel") or "").lower() + if roleLabel.endswith("-admin"): + adminInstRoleId = ir.get("id") + break + if not adminInstRoleId: + raise ValueError( + f"No feature-specific admin role (e.g. {featureCode}-admin) for instance {instanceId}. " + f"Template roles not synced for feature '{featureCode}'." + ) + self.createFeatureAccess(userId, instanceId, roleIds=[adminInstRoleId]) + except Exception as e: + logger.error(f"Error auto-creating instance for '{featureName}': {e}") + + logger.info(f"Provisioned mandate {mandateId} (plan={planKey}) for user {userId}, instances={createdInstances}") + return { + "mandateId": mandateId, + "planKey": planKey, + "featureInstances": createdInstances, + } + except Exception as e: + logger.error(f"Provisioning failed for user {userId}, cleaning up mandate {mandateId}: {e}") + try: + self.db.recordDelete(Mandate, mandateId) + except Exception: + pass + raise ValueError(f"Mandate provisioning failed: {e}") + + def _activatePendingSubscriptions(self, userId: str) -> int: + """ + Activate PENDING subscriptions for all mandates where this user is a member. + Called on login — trial period begins NOW, not at registration. + Uses the subscription interface (poweron_billing) for all subscription operations. + Returns number of activated subscriptions. + """ + from modules.datamodels.datamodelSubscription import ( + SubscriptionStatusEnum, BUILTIN_PLANS, + ) + from modules.interfaces.interfaceDbSubscription import _getRootInterface as _getSubRoot + from datetime import datetime, timezone, timedelta + + activated = 0 + subInterface = _getSubRoot() + + userMandates = self.db.getRecordset( + UserMandate, recordFilter={"userId": userId, "enabled": True} + ) + + for um in userMandates: + mandateId = um.get("mandateId") + allSubs = subInterface.listForMandate(mandateId) + pendingSubs = [s for s in allSubs if s.get("status") == SubscriptionStatusEnum.PENDING.value] + + for sub in pendingSubs: + subId = sub.get("id") + planKey = sub.get("planKey") + plan = BUILTIN_PLANS.get(planKey) + now = datetime.now(timezone.utc) + + targetStatus = SubscriptionStatusEnum.TRIALING if plan and plan.trialDays else SubscriptionStatusEnum.ACTIVE + additionalData = { + "currentPeriodStart": now.isoformat(), + } + + if plan and plan.trialDays: + trialEnd = now + timedelta(days=plan.trialDays) + additionalData["trialEndsAt"] = trialEnd.isoformat() + additionalData["currentPeriodEnd"] = trialEnd.isoformat() + elif plan and plan.billingPeriod: + from modules.datamodels.datamodelSubscription import BillingPeriodEnum + if plan.billingPeriod == BillingPeriodEnum.MONTHLY: + additionalData["currentPeriodEnd"] = (now + timedelta(days=30)).isoformat() + elif plan.billingPeriod == BillingPeriodEnum.YEARLY: + additionalData["currentPeriodEnd"] = (now + timedelta(days=365)).isoformat() + + try: + subInterface.transitionStatus( + subId, + expectedFromStatus=SubscriptionStatusEnum.PENDING, + toStatus=targetStatus, + additionalData=additionalData, + ) + activated += 1 + logger.info(f"Activated subscription {subId} (plan={planKey}) for mandate {mandateId}: {targetStatus.value}") + except Exception as e: + logger.error(f"Failed to activate subscription {subId}: {e}") + + return activated + def updateMandate(self, mandateId: str, updateData: Dict[str, Any]) -> Mandate: """Updates a mandate if user has access.""" try: @@ -1493,39 +1609,203 @@ class AppObjects: logger.error(f"Error updating mandate: {str(e)}") raise ValueError(f"Failed to update mandate: {str(e)}") - def deleteMandate(self, mandateId: str) -> bool: - """Deletes a mandate if user has access. System mandates cannot be deleted.""" + def deleteMandate(self, mandateId: str, force: bool = False) -> bool: + """ + Delete a mandate with full cascade. + + Default (force=False): soft-delete — sets enabled=false. + With force=True: hard-delete — removes all related data. + System mandates (isSystem=True) cannot be deleted. + """ try: - # Check if mandate exists and user has access mandate = self.getMandate(mandateId) if not mandate: return False - # System mandates (isSystem=True) cannot be deleted if getattr(mandate, "isSystem", False): raise ValueError(f"System mandate '{mandate.name}' cannot be deleted") if not self.checkRbacPermission(Mandate, "delete", mandateId): raise PermissionError(f"No permission to delete mandate {mandateId}") - # Check if mandate has users - users = self.getUsersByMandate(mandateId) - if users: - raise ValueError( - f"Cannot delete mandate {mandateId} with existing users" - ) + if not force: + from modules.shared.timeUtils import getUtcTimestamp + self.db.recordModify(Mandate, mandateId, {"enabled": False, "deletedAt": getUtcTimestamp()}) + logger.info(f"Soft-deleted mandate {mandateId} (30-day retention)") + return True - # Delete mandate + # Hard delete with cascade + from modules.datamodels.datamodelSubscription import MandateSubscription + from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog + from modules.datamodels.datamodelFiles import FileItem + from modules.datamodels.datamodelDataSource import DataSource + from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk + from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource + from modules.datamodels.datamodelBilling import BillingSettings, BillingAccount, BillingTransaction + from modules.datamodels.datamodelRbac import FeatureAccessRole, UserMandateRole + from modules.features.neutralization.datamodelFeatureNeutralizer import DataNeutralizerAttributes + + instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) + + # 0. Delete instance-scoped data for each FeatureInstance + for inst in instances: + instId = inst.get("id") + if not instId: + continue + + # 0a. ContentChunk (embeddings) + FileContentIndex (knowledge/RAG) + fciRecords = self.db.getRecordset(FileContentIndex, recordFilter={"featureInstanceId": instId}) + for rec in fciRecords: + chunks = self.db.getRecordset(ContentChunk, recordFilter={"fileContentIndexId": rec.get("id")}) + for chunk in chunks: + self.db.recordDelete(ContentChunk, chunk.get("id")) + self.db.recordDelete(FileContentIndex, rec.get("id")) + if fciRecords: + logger.info(f"Cascade: deleted {len(fciRecords)} FileContentIndex records (with chunks) for instance {instId}") + + # 0b. DataNeutralizerAttributes + dnaRecords = self.db.getRecordset(DataNeutralizerAttributes, recordFilter={"featureInstanceId": instId}) + for rec in dnaRecords: + self.db.recordDelete(DataNeutralizerAttributes, rec.get("id")) + if dnaRecords: + logger.info(f"Cascade: deleted {len(dnaRecords)} DataNeutralizerAttributes for instance {instId}") + + # 0c. DataSource + dsRecords = self.db.getRecordset(DataSource, recordFilter={"featureInstanceId": instId}) + for rec in dsRecords: + self.db.recordDelete(DataSource, rec.get("id")) + if dsRecords: + logger.info(f"Cascade: deleted {len(dsRecords)} DataSource records for instance {instId}") + + # 0c2. FeatureDataSource + fdsRecords = self.db.getRecordset(FeatureDataSource, recordFilter={"featureInstanceId": instId}) + for rec in fdsRecords: + self.db.recordDelete(FeatureDataSource, rec.get("id")) + if fdsRecords: + logger.info(f"Cascade: deleted {len(fdsRecords)} FeatureDataSource records for instance {instId}") + + # 0d. FileItem + fileRecords = self.db.getRecordset(FileItem, recordFilter={"featureInstanceId": instId}) + for rec in fileRecords: + self.db.recordDelete(FileItem, rec.get("id")) + if fileRecords: + logger.info(f"Cascade: deleted {len(fileRecords)} FileItem records for instance {instId}") + + # 0e. ChatWorkflow + ChatMessage + ChatLog + workflows = self.db.getRecordset(ChatWorkflow, recordFilter={"featureInstanceId": instId}) + for wf in workflows: + wfId = wf.get("id") + if not wfId: + continue + msgs = self.db.getRecordset(ChatMessage, recordFilter={"workflowId": wfId}) + for msg in msgs: + self.db.recordDelete(ChatMessage, msg.get("id")) + logs = self.db.getRecordset(ChatLog, recordFilter={"workflowId": wfId}) + for log in logs: + self.db.recordDelete(ChatLog, log.get("id")) + self.db.recordDelete(ChatWorkflow, wfId) + if workflows: + logger.info(f"Cascade: deleted {len(workflows)} ChatWorkflows (with messages/logs) for instance {instId}") + + # 1. Delete FeatureAccess + FeatureAccessRole for all instances + for inst in instances: + instId = inst.get("id") + accesses = self.db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instId}) + for access in accesses: + roles = self.db.getRecordset(FeatureAccessRole, recordFilter={"featureAccessId": access.get("id")}) + for role in roles: + self.db.recordDelete(FeatureAccessRole, role.get("id")) + self.db.recordDelete(FeatureAccess, access.get("id")) + self.db.recordDelete(FeatureInstance, instId) + logger.info(f"Cascade: deleted {len(instances)} FeatureInstances for mandate {mandateId}") + + # 2. Delete UserMandate + UserMandateRole + memberships = self.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) + for um in memberships: + umRoles = self.db.getRecordset(UserMandateRole, recordFilter={"userMandateId": um.get("id")}) + for umr in umRoles: + self.db.recordDelete(UserMandateRole, umr.get("id")) + self.db.recordDelete(UserMandate, um.get("id")) + logger.info(f"Cascade: deleted {len(memberships)} UserMandates for mandate {mandateId}") + + # 3. Cancel Stripe subscriptions + delete MandateSubscription records + subs = self.db.getRecordset(MandateSubscription, recordFilter={"mandateId": mandateId}) + for sub in subs: + subId = sub.get("id") + stripeSubId = sub.get("stripeSubscriptionId") + if stripeSubId: + try: + from modules.shared.stripeClient import getStripeClient + stripe = getStripeClient() + stripe.Subscription.cancel(stripeSubId) + logger.info(f"Cancelled Stripe subscription {stripeSubId} for mandate {mandateId}") + except Exception as e: + logger.warning(f"Failed to cancel Stripe sub {stripeSubId}: {e}") + self.db.recordDelete(MandateSubscription, subId) + logger.info(f"Cascade: deleted {len(subs)} subscriptions for mandate {mandateId}") + + # 3b. Delete Billing data + billingTxs = self.db.getRecordset(BillingTransaction, recordFilter={"mandateId": mandateId}) if hasattr(BillingTransaction, '__table_name__') else [] + billingAccounts = self.db.getRecordset(BillingAccount, recordFilter={"mandateId": mandateId}) + for acc in billingAccounts: + accTxs = self.db.getRecordset(BillingTransaction, recordFilter={"accountId": acc.get("id")}) + for tx in accTxs: + self.db.recordDelete(BillingTransaction, tx.get("id")) + self.db.recordDelete(BillingAccount, acc.get("id")) + billingSettings = self.db.getRecordset(BillingSettings, recordFilter={"mandateId": mandateId}) + for bs in billingSettings: + self.db.recordDelete(BillingSettings, bs.get("id")) + if billingAccounts or billingSettings: + logger.info(f"Cascade: deleted billing data for mandate {mandateId}") + + # 4. Delete mandate-level Roles + from modules.datamodels.datamodelRbac import Role, AccessRule + roles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId}) + for role in roles: + rules = self.db.getRecordset(AccessRule, recordFilter={"roleId": role.get("id")}) + for rule in rules: + self.db.recordDelete(AccessRule, rule.get("id")) + self.db.recordDelete(Role, role.get("id")) + logger.info(f"Cascade: deleted {len(roles)} Roles for mandate {mandateId}") + + # 5. Delete mandate record success = self.db.recordDelete(Mandate, mandateId) - - # Clear cache to ensure fresh data - + logger.info(f"Hard-deleted mandate {mandateId}") return success except Exception as e: logger.error(f"Error deleting mandate: {str(e)}") raise ValueError(f"Failed to delete mandate: {str(e)}") + def restoreMandate(self, mandateId: str) -> bool: + """Restore a soft-deleted mandate (undo soft-delete within the 30-day retention window).""" + mandate = self.getMandate(mandateId) + if not mandate: + return False + self.db.recordModify(Mandate, mandateId, {"enabled": True, "deletedAt": None}) + logger.info(f"Restored soft-deleted mandate {mandateId}") + return True + + def purgeExpiredMandates(self, retentionDays: int = 30) -> int: + """Hard-delete all mandates whose soft-delete timestamp exceeds the retention period.""" + import time + cutoff = time.time() - (retentionDays * 86400) + allMandates = self.db.getRecordset(Mandate) + purged = 0 + for m in allMandates: + deletedAt = m.get("deletedAt") if isinstance(m, dict) else getattr(m, "deletedAt", None) + enabled = m.get("enabled") if isinstance(m, dict) else getattr(m, "enabled", True) + mandateId = m.get("id") if isinstance(m, dict) else getattr(m, "id", None) + if deletedAt and not enabled and deletedAt < cutoff and mandateId: + try: + self.deleteMandate(mandateId, force=True) + purged += 1 + except Exception as e: + logger.error(f"Failed to purge expired mandate {mandateId}: {e}") + if purged: + logger.info(f"Purged {purged} expired mandate(s) beyond {retentionDays}-day retention") + return purged + # ============================================ # User-Mandate Membership Methods (Multi-Tenant) # ============================================ @@ -1548,7 +1828,7 @@ class AppObjects: ) if not records: return None - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return UserMandate(**cleanedRecord) except Exception as e: logger.error(f"Error getting UserMandate: {e}") @@ -1571,60 +1851,62 @@ class AppObjects: ) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(UserMandate(**cleanedRecord)) return result except Exception as e: logger.error(f"Error getting UserMandates: {e}") return [] - def createUserMandate(self, userId: str, mandateId: str, roleIds: List[str] = None) -> UserMandate: + def createUserMandate(self, userId: str, mandateId: str, roleIds: List[str] = None, *, skipCapacityCheck: bool = False) -> UserMandate: """ Create a UserMandate record (add user to mandate). - Also creates a billing account for the user if billing is configured for PREPAY_USER. + Also creates a billing audit account for the user if billing is configured. + + INVARIANT: A UserMandate MUST have at least one UserMandateRole. Args: userId: User ID mandateId: Mandate ID - roleIds: Optional list of role IDs to assign + roleIds: List of role IDs to assign (at least one required) + skipCapacityCheck: If True, skip subscription capacity check (used during initial provisioning + when the subscription hasn't been created yet) Returns: Created UserMandate object """ + if not roleIds: + raise ValueError(f"Cannot create UserMandate without roles for user {userId} in mandate {mandateId}") + try: - # Check if already exists existing = self.getUserMandate(userId, mandateId) if existing: raise ValueError(f"User {userId} is already member of mandate {mandateId}") - # Subscription capacity check (before insert) - self._checkSubscriptionCapacity(mandateId, "users", delta=1) + if not skipCapacityCheck: + self._checkSubscriptionCapacity(mandateId, "users", delta=1) - # Create UserMandate userMandate = UserMandate( userId=userId, mandateId=mandateId, enabled=True ) createdRecord = self.db.recordCreate(UserMandate, userMandate.model_dump()) + if not createdRecord: + raise ValueError("Database failed to create UserMandate record") - # Assign roles via junction table - if roleIds and createdRecord: - userMandateId = createdRecord.get("id") - for roleId in roleIds: - userMandateRole = UserMandateRole( - userMandateId=userMandateId, - roleId=roleId - ) - self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) + userMandateId = createdRecord.get("id") + for roleId in roleIds: + userMandateRole = UserMandateRole( + userMandateId=userMandateId, + roleId=roleId + ) + self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) - # Create billing account for user if billing is configured self._ensureUserBillingAccount(userId, mandateId) - - # Sync Stripe quantity after successful insert self._syncSubscriptionQuantity(mandateId) - cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + cleanedRecord = dict(createdRecord) return UserMandate(**cleanedRecord) except Exception as e: logger.error(f"Error creating UserMandate: {e}") @@ -1632,43 +1914,20 @@ class AppObjects: def _ensureUserBillingAccount(self, userId: str, mandateId: str) -> None: """ - Ensure a user has a billing account for the mandate if billing is configured. - User accounts are always created for all billing models (for audit trail). - Initial balance depends on billing model: - - PREPAY_USER: defaultUserCredit from mandate BillingSettings when joining the root mandate (missing key => 0.0); - other mandates get 0.0. - - PREPAY_MANDATE: 0.0 on the user account (shared pool — no per-user start credit) - - Args: - userId: User ID - mandateId: Mandate ID + Ensure a user has a billing audit account for the mandate. + Balance is always on the mandate pool (PREPAY_MANDATE). User accounts are for audit trail only. """ try: from modules.interfaces.interfaceDbBilling import _getRootInterface as getBillingRootInterface - from modules.datamodels.datamodelBilling import BillingModelEnum, parseBillingModelFromStoredValue billingInterface = getBillingRootInterface() settings = billingInterface.getSettings(mandateId) if not settings: - return # No billing configured for this mandate + return - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - # Initial balance depends on billing model (start credit only on root mandate for PREPAY_USER) - rootMandateId = self._getRootMandateId() - isRootMandate = rootMandateId is not None and str(mandateId) == str(rootMandateId) - if billingModel == BillingModelEnum.PREPAY_USER: - initialBalance = ( - float(settings.get("defaultUserCredit", 0.0)) - if isRootMandate - else 0.0 - ) - else: - initialBalance = 0.0 # PREPAY_MANDATE: budget is on pool - - billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=initialBalance) - logger.info(f"Ensured billing account for user {userId} in mandate {mandateId} (model={billingModel.value}, initial={initialBalance} CHF)") + billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=0.0) + logger.info(f"Ensured billing audit account for user {userId} in mandate {mandateId}") except Exception as e: logger.warning(f"Failed to create billing account for user {userId} (non-critical): {e}") @@ -1685,14 +1944,26 @@ class AppObjects: raise logger.debug(f"Subscription capacity check skipped: {e}") - def _syncSubscriptionQuantity(self, mandateId: str) -> None: - """Sync Stripe subscription quantities after a resource mutation.""" + def _syncSubscriptionQuantity(self, mandateId: str, *, raiseOnError: bool = False) -> None: + """Sync Stripe subscription quantities after a resource mutation. + + Args: + raiseOnError: If True, propagate errors (billing-critical paths). + """ try: from modules.interfaces.interfaceDbSubscription import getInterface as getSubInterface from modules.security.rootAccess import getRootUser subIf = getSubInterface(getRootUser(), mandateId) - subIf.syncQuantityToStripe(mandateId) + operative = subIf.getOperativeForMandate(mandateId) + if not operative: + if raiseOnError: + raise ValueError(f"Kein operatives Abonnement für Mandant {mandateId}") + logger.debug("No operative subscription for mandate %s — quantity sync skipped", mandateId) + return + subIf.syncQuantityToStripe(operative["id"], raiseOnError=raiseOnError) except Exception as e: + if raiseOnError: + raise logger.debug(f"Subscription quantity sync skipped: {e}") def deleteUserMandate(self, userId: str, mandateId: str) -> bool: @@ -1754,7 +2025,7 @@ class AppObjects: ) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(UserMandate(**cleanedRecord)) return result except Exception as e: @@ -1778,7 +2049,7 @@ class AppObjects: ) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(UserMandateRole(**cleanedRecord)) return result except Exception as e: @@ -1875,7 +2146,7 @@ class AppObjects: recordFilter={"userMandateId": userMandateId, "roleId": roleId} ) if existing: - cleanedRecord = {k: v for k, v in existing[0].items() if not k.startswith("_")} + cleanedRecord = dict(existing[0]) return UserMandateRole(**cleanedRecord) userMandateRole = UserMandateRole( @@ -1883,7 +2154,7 @@ class AppObjects: roleId=roleId ) createdRecord = self.db.recordCreate(UserMandateRole, userMandateRole.model_dump()) - cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + cleanedRecord = dict(createdRecord) return UserMandateRole(**cleanedRecord) except Exception as e: logger.error(f"Error adding role to UserMandate: {e}") @@ -1948,7 +2219,7 @@ class AppObjects: ) if not records: return None - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return FeatureAccess(**cleanedRecord) except Exception as e: logger.error(f"Error getting FeatureAccess: {e}") @@ -1971,7 +2242,7 @@ class AppObjects: ) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(FeatureAccess(**cleanedRecord)) return result except Exception as e: @@ -1995,7 +2266,7 @@ class AppObjects: ) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(FeatureAccess(**cleanedRecord)) return result except Exception as e: @@ -2007,42 +2278,44 @@ class AppObjects: Create a FeatureAccess record (grant user access to feature instance). Also auto-assigns the user to the mandate with the 'user' role if not already a member. + INVARIANT: A FeatureAccess MUST have at least one FeatureAccessRole. + Args: userId: User ID featureInstanceId: FeatureInstance ID - roleIds: Optional list of role IDs to assign + roleIds: List of role IDs to assign (at least one required) Returns: Created FeatureAccess object """ + if not roleIds: + raise ValueError(f"Cannot create FeatureAccess without roles for user {userId} on instance {featureInstanceId}") + try: - # Check if already exists existing = self.getFeatureAccess(userId, featureInstanceId) if existing: raise ValueError(f"User {userId} already has access to feature instance {featureInstanceId}") - # Auto-assign user to mandate with 'user' role if not already a member self._ensureUserMandateMembership(userId, featureInstanceId) - # Create FeatureAccess featureAccess = FeatureAccess( userId=userId, featureInstanceId=featureInstanceId, enabled=True ) createdRecord = self.db.recordCreate(FeatureAccess, featureAccess.model_dump()) + if not createdRecord: + raise ValueError("Database failed to create FeatureAccess record") - # Assign roles via junction table - if roleIds and createdRecord: - featureAccessId = createdRecord.get("id") - for roleId in roleIds: - featureAccessRole = FeatureAccessRole( - featureAccessId=featureAccessId, - roleId=roleId - ) - self.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + featureAccessId = createdRecord.get("id") + for roleId in roleIds: + featureAccessRole = FeatureAccessRole( + featureAccessId=featureAccessId, + roleId=roleId + ) + self.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) - cleanedRecord = {k: v for k, v in createdRecord.items() if not k.startswith("_")} + cleanedRecord = dict(createdRecord) return FeatureAccess(**cleanedRecord) except Exception as e: logger.error(f"Error creating FeatureAccess: {e}") @@ -2051,7 +2324,7 @@ class AppObjects: def _ensureUserMandateMembership(self, userId: str, featureInstanceId: str) -> None: """ Ensure user is a member of the mandate that owns the feature instance. - If not already a member, adds them with the 'user' role (no access rights, membership only). + If not already a member, adds them with the 'user' role. """ try: from modules.interfaces.interfaceFeatures import getFeatureInterface @@ -2064,28 +2337,30 @@ class AppObjects: mandateId = str(instance.mandateId) - # Check if user already has mandate membership existing = self.getUserMandate(userId, mandateId) if existing: logger.debug(f"User {userId} already member of mandate {mandateId}") return - # Find the mandate-level 'user' role (membership marker, no access rights) userRoles = self.db.getRecordset( Role, recordFilter={"roleLabel": "user", "mandateId": mandateId, "featureInstanceId": None} ) userRoleId = userRoles[0].get("id") if userRoles else None - roleIds = [userRoleId] if userRoleId else [] + if not userRoleId: + raise ValueError(f"No 'user' role found for mandate {mandateId} — cannot assign user without role") - self.createUserMandate(userId, mandateId, roleIds) + self.createUserMandate(userId, mandateId, roleIds=[userRoleId]) logger.info(f"Auto-assigned user {userId} to mandate {mandateId} with 'user' role (via feature instance {featureInstanceId})") - except ValueError: - # createUserMandate raises ValueError if already exists - safe to ignore - pass + except ValueError as ve: + if "already member" in str(ve): + pass + else: + raise except Exception as e: logger.error(f"Error auto-assigning user {userId} to mandate: {e}") + raise def getRoleIdsForFeatureAccess(self, featureAccessId: str) -> List[str]: """ @@ -2178,7 +2453,7 @@ class AppObjects: try: records = self.db.getRecordset(Invitation, recordFilter={"id": invitationId}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return Invitation(**cleanedRecord) return None except Exception as e: @@ -2198,7 +2473,7 @@ class AppObjects: try: records = self.db.getRecordset(Invitation, recordFilter={"token": token}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return Invitation(**cleanedRecord) return None except Exception as e: @@ -2219,7 +2494,7 @@ class AppObjects: records = self.db.getRecordset(Invitation, recordFilter={"mandateId": mandateId}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Invitation(**cleanedRecord)) return result except Exception as e: @@ -2237,10 +2512,10 @@ class AppObjects: List of Invitation objects """ try: - records = self.db.getRecordset(Invitation, recordFilter={"createdBy": creatorId}) + records = self.db.getRecordset(Invitation, recordFilter={"sysCreatedBy": creatorId}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Invitation(**cleanedRecord)) return result except Exception as e: @@ -2261,7 +2536,7 @@ class AppObjects: records = self.db.getRecordset(Invitation, recordFilter={"usedBy": usedById}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Invitation(**cleanedRecord)) return result except Exception as e: @@ -2282,13 +2557,25 @@ class AppObjects: records = self.db.getRecordset(Invitation, recordFilter={"targetUsername": targetUsername}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Invitation(**cleanedRecord)) return result except Exception as e: logger.error(f"Error getting invitations for target username {targetUsername}: {e}") return [] + def getInvitationsByEmail(self, email: str) -> List[Invitation]: + """Get all invitations for a target email address (email-only invitations).""" + try: + records = self.db.getRecordset(Invitation, recordFilter={"email": email}) + result = [] + for record in records: + result.append(Invitation(**dict(record))) + return result + except Exception as e: + logger.error(f"Error getting invitations for email {email}: {e}") + return [] + # ============================================ # Additional Helper Methods # ============================================ @@ -2309,13 +2596,10 @@ class AppObjects: items = [] for record in result["items"]: - cleanedRecord = { - k: v for k, v in record.items() - if not k.startswith("_") and k not in ["hashedPassword", "resetToken", "resetTokenExpires"] - } - if cleanedRecord.get("roleLabels") is None: - cleanedRecord["roleLabels"] = [] - items.append(User(**cleanedRecord)) + user = User.model_validate(record) + if user.roleLabels is None: + user = user.model_copy(update={"roleLabels": []}) + items.append(user) if pagination is None: return items @@ -2344,7 +2628,7 @@ class AppObjects: try: records = self.db.getRecordset(UserMandate, recordFilter={"id": userMandateId}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return UserMandate(**cleanedRecord) return None except Exception as e: @@ -2365,7 +2649,7 @@ class AppObjects: records = self.db.getRecordset(UserMandateRole, recordFilter={"roleId": roleId}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(UserMandateRole(**cleanedRecord)) return result except Exception as e: @@ -2385,7 +2669,7 @@ class AppObjects: try: records = self.db.getRecordset(FeatureInstance, recordFilter={"id": instanceId}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return FeatureInstance(**cleanedRecord) return None except Exception as e: @@ -2405,7 +2689,7 @@ class AppObjects: try: records = self.db.getRecordset(Feature, recordFilter={"code": featureCode}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return Feature(**cleanedRecord) return None except Exception as e: @@ -2430,7 +2714,7 @@ class AppObjects: records = self.db.getRecordset(FeatureInstance, recordFilter=recordFilter) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(FeatureInstance(**cleanedRecord)) return result except Exception as e: @@ -2454,7 +2738,7 @@ class AppObjects: try: records = self.db.getRecordset(UserNotification, recordFilter={"id": notificationId}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return UserNotification(**cleanedRecord) return None except Exception as e: @@ -2485,10 +2769,10 @@ class AppObjects: records = self.db.getRecordset(UserNotification, recordFilter=recordFilter) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(UserNotification(**cleanedRecord)) - # Sort by createdAt descending - result.sort(key=lambda x: x.createdAt or 0, reverse=True) + # Sort by sysCreatedAt descending + result.sort(key=lambda x: x.sysCreatedAt or 0, reverse=True) if limit: result = result[:limit] return result @@ -2513,7 +2797,7 @@ class AppObjects: try: records = self.db.getRecordset(AccessRule, recordFilter={"id": ruleId}) if records: - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return AccessRule(**cleanedRecord) return None except Exception as e: @@ -2534,7 +2818,7 @@ class AppObjects: records = self.db.getRecordset(AccessRule, recordFilter={"roleId": roleId}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(AccessRule(**cleanedRecord)) return result except Exception as e: @@ -2555,7 +2839,7 @@ class AppObjects: records = self.db.getRecordset(Role, recordFilter={"featureInstanceId": featureInstanceId}) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Role(**cleanedRecord)) return result except Exception as e: @@ -2580,7 +2864,7 @@ class AppObjects: records = self.db.getRecordset(Role, recordFilter=recordFilter) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Role(**cleanedRecord)) return result except Exception as e: @@ -2618,8 +2902,8 @@ class AppObjects: # Ensure token has required fields if not token.id: token.id = str(uuid.uuid4()) - if not token.createdAt: - token.createdAt = getUtcTimestamp() + if not token.sysCreatedAt: + token.sysCreatedAt = getUtcTimestamp() # If replace_existing is True, delete old access tokens for this user and authority first if replace_existing: @@ -2652,12 +2936,7 @@ class AppObjects: ) # Continue with saving the new token even if deletion fails - # Convert to dict and ensure all fields are properly set token_dict = token.model_dump() - # Ensure userId is set to current user - # Convert to dict and ensure all fields are properly set - token_dict = token.model_dump() - # Ensure userId is set to current user token_dict["userId"] = self.currentUser.id # Save to database @@ -2696,8 +2975,8 @@ class AppObjects: # Ensure token has required fields if not token.id: token.id = str(uuid.uuid4()) - if not token.createdAt: - token.createdAt = getUtcTimestamp() + if not token.sysCreatedAt: + token.sysCreatedAt = getUtcTimestamp() # Convert to dict and ensure all fields are properly set token_dict = token.model_dump() @@ -2779,7 +3058,7 @@ class AppObjects: ) result = [] for token_dict in tokens: - cleanedRecord = {k: v for k, v in token_dict.items() if not k.startswith("_")} + cleanedRecord = dict(token_dict) result.append(Token(**cleanedRecord)) return result except Exception as e: @@ -2800,7 +3079,7 @@ class AppObjects: ) result = [] for token_dict in tokens: - cleanedRecord = {k: v for k, v in token_dict.items() if not k.startswith("_")} + cleanedRecord = dict(token_dict) result.append(Token(**cleanedRecord)) return result except Exception as e: @@ -3114,7 +3393,7 @@ class AppObjects: # Filter out database-specific fields filteredRules = [] for rule in rules: - cleanedRule = {k: v for k, v in rule.items() if not k.startswith("_")} + cleanedRule = dict(rule) filteredRules.append(cleanedRule) # If no pagination requested, return all items @@ -3298,7 +3577,7 @@ class AppObjects: Role, recordFilter={"mandateId": mandateId, "featureInstanceId": None} ) - return [Role(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in roles] + return [Role(**dict(r)) for r in roles] except Exception as e: logger.error(f"Error getting roles for mandate {mandateId}: {e}") return [] @@ -3319,7 +3598,7 @@ class AppObjects: items = [] for record in result["items"]: - cleanedRole = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRole = dict(record) items.append(Role(**cleanedRole)) if pagination is None: diff --git a/modules/interfaces/interfaceDbBilling.py b/modules/interfaces/interfaceDbBilling.py index 2db71bb4..948f8918 100644 --- a/modules/interfaces/interfaceDbBilling.py +++ b/modules/interfaces/interfaceDbBilling.py @@ -9,7 +9,7 @@ All billing data is stored in the poweron_billing database. import logging from typing import Dict, Any, List, Optional, Union -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, timezone import uuid from modules.connectors.connectorDbPostgre import DatabaseConnector @@ -24,19 +24,49 @@ from modules.datamodels.datamodelBilling import ( BillingSettings, StripeWebhookEvent, UsageStatistics, - BillingModelEnum, - AccountTypeEnum, TransactionTypeEnum, ReferenceTypeEnum, PeriodTypeEnum, BillingBalanceResponse, BillingCheckResult, - parseBillingModelFromStoredValue, + STORAGE_PRICE_PER_GB_CHF, ) logger = logging.getLogger(__name__) +def _logBillingTransactionsMissingSysCreatedAt(rows: List[Dict[str, Any]], context: str) -> None: + """Log ERROR when sysCreatedAt is missing; does not raise.""" + missingIds = [r.get("id") for r in rows if r.get("sysCreatedAt") is None] + if not missingIds: + return + cap = 40 + sample = missingIds[:cap] + suffix = f"; ... (+{len(missingIds) - cap} more)" if len(missingIds) > cap else "" + logger.error( + "BillingTransaction missing sysCreatedAt (%s): count=%s; transactionIds=%s%s", + context, + len(missingIds), + sample, + suffix, + ) + + +def _numericSysCreatedAtForSort(row: Dict[str, Any]) -> float: + v = row["sysCreatedAt"] + if isinstance(v, datetime): + return v.timestamp() + return float(v) + + +def _sortBillingTransactionsBySysCreatedAtDesc(rows: List[Dict[str, Any]], context: str) -> None: + _logBillingTransactionsMissingSysCreatedAt(rows, context) + valid = [r for r in rows if r.get("sysCreatedAt") is not None] + invalid = [r for r in rows if r.get("sysCreatedAt") is None] + valid.sort(key=_numericSysCreatedAtForSort, reverse=True) + rows[:] = valid + invalid + + def _getAppDatabaseConnector() -> DatabaseConnector: """App DB connector (same config as UserMandate reads in this module).""" return DatabaseConnector( @@ -160,8 +190,6 @@ class BillingObjects: """ Get billing settings for a mandate. - Normalizes billingModel for API (legacy UNLIMITED → PREPAY_MANDATE) and persists once. - Args: mandateId: Mandate ID @@ -175,27 +203,7 @@ class BillingObjects: ) if not results: return None - row = dict(results[0]) - raw_bm = row.get("billingModel") - parsed = parseBillingModelFromStoredValue(raw_bm) - if str(raw_bm or "").strip().upper() == "UNLIMITED": - try: - self.updateSettings( - row["id"], - {"billingModel": BillingModelEnum.PREPAY_MANDATE.value}, - ) - logger.info( - "Migrated billing settings for mandate %s: UNLIMITED → PREPAY_MANDATE", - mandateId, - ) - except Exception as mig_err: - logger.warning( - "Could not persist billing model migration for mandate %s: %s", - mandateId, - mig_err, - ) - row["billingModel"] = parsed.value - return row + return dict(results[0]) except Exception as e: logger.error(f"Error getting billing settings: {e}") return None @@ -226,13 +234,12 @@ class BillingObjects: """ return self.db.recordModify(BillingSettings, settingsId, updates) - def getOrCreateSettings(self, mandateId: str, defaultModel: BillingModelEnum = BillingModelEnum.PREPAY_MANDATE) -> Dict[str, Any]: + def getOrCreateSettings(self, mandateId: str) -> Dict[str, Any]: """ Get or create billing settings for a mandate. Args: mandateId: Mandate ID - defaultModel: Default billing model if creating Returns: BillingSettings dict @@ -243,8 +250,6 @@ class BillingObjects: settings = BillingSettings( mandateId=mandateId, - billingModel=defaultModel, - defaultUserCredit=0.0, warningThresholdPercent=10.0, notifyOnWarning=True, ) @@ -281,7 +286,7 @@ class BillingObjects: BillingAccount, recordFilter={ "mandateId": mandateId, - "accountType": AccountTypeEnum.MANDATE.value + "userId": None } ) return results[0] if results else None @@ -305,8 +310,7 @@ class BillingObjects: BillingAccount, recordFilter={ "mandateId": mandateId, - "userId": userId, - "accountType": AccountTypeEnum.USER.value + "userId": userId } ) return results[0] if results else None @@ -376,7 +380,6 @@ class BillingObjects: account = BillingAccount( mandateId=mandateId, - accountType=AccountTypeEnum.MANDATE, balance=initialBalance, enabled=True ) @@ -401,7 +404,6 @@ class BillingObjects: account = BillingAccount( mandateId=mandateId, userId=userId, - accountType=AccountTypeEnum.USER, balance=initialBalance, enabled=True ) @@ -422,7 +424,7 @@ class BillingObjects: def ensureAllMandateSettingsExist(self) -> int: """ Efficiently ensure all mandates have billing settings. - Creates default settings (PREPAY_MANDATE, 0 CHF) for mandates without settings. + Creates default settings (0 CHF) for mandates without settings. Uses bulk queries to minimize database connections. Returns: @@ -451,16 +453,13 @@ class BillingObjects: if not mandateId or mandateId in existingMandateIds: continue - # Create default billing settings settings = BillingSettings( mandateId=mandateId, - billingModel=BillingModelEnum.PREPAY_MANDATE, - defaultUserCredit=0.0, warningThresholdPercent=10.0, notifyOnWarning=True, ) self.createSettings(settings) - existingMandateIds.add(mandateId) # Track newly created + existingMandateIds.add(mandateId) settingsCreated += 1 if settingsCreated > 0: @@ -475,11 +474,7 @@ class BillingObjects: def ensureAllUserAccountsExist(self) -> int: """ Ensure all users across all mandates have billing accounts. - User accounts are always created regardless of billing model (for audit trail). - Initial balance depends on billing model: - - PREPAY_USER: defaultUserCredit from settings only for the root mandate; other mandates get 0.0 - - PREPAY_MANDATE: 0.0 (budget is on pool) - + User accounts are always created for audit trail with initial balance 0.0. Uses bulk queries to minimize database connections. Returns: @@ -488,44 +483,29 @@ class BillingObjects: try: accountsCreated = 0 appDb = _getAppDatabaseConnector() - rootMandateId = _getCachedRootMandateId() - # Step 1: Get all billing settings (all mandates with settings get user accounts) allSettings = self.db.getRecordset(BillingSettings) - billingMandates = {} # mandateId -> (billingModel, defaultCredit) - for s in allSettings: - billingModel = parseBillingModelFromStoredValue(s.get("billingModel")).value - mid = s.get("mandateId") - isRoot = rootMandateId is not None and str(mid) == str(rootMandateId) - if billingModel == BillingModelEnum.PREPAY_USER.value: - defaultCredit = ( - float(s.get("defaultUserCredit", 0.0) or 0.0) if isRoot else 0.0 - ) - else: - defaultCredit = 0.0 - billingMandates[mid] = (billingModel, defaultCredit) + billingMandateIds = set( + s.get("mandateId") for s in allSettings if s.get("mandateId") + ) - if not billingMandates: + if not billingMandateIds: logger.debug("No billable mandates found, skipping account check") return 0 - # Step 2: Get all existing USER accounts in one query - allAccounts = self.db.getRecordset( - BillingAccount, - recordFilter={"accountType": AccountTypeEnum.USER.value} - ) + allAccounts = self.db.getRecordset(BillingAccount) existingAccountKeys = set() for acc in allAccounts: + if not acc.get("userId"): + continue key = (acc.get("mandateId"), acc.get("userId")) existingAccountKeys.add(key) - # Step 3: Get all user-mandate combinations from APP database allUserMandates = appDb.getRecordset( UserMandate, recordFilter={"enabled": True} ) - # Step 4: Create missing accounts for um in allUserMandates: mandateId = um.get("mandateId") userId = um.get("userId") @@ -533,32 +513,20 @@ class BillingObjects: if not mandateId or not userId: continue - if mandateId not in billingMandates: + if mandateId not in billingMandateIds: continue key = (mandateId, userId) if key in existingAccountKeys: continue - billingModel, defaultCredit = billingMandates[mandateId] - account = BillingAccount( mandateId=mandateId, userId=userId, - accountType=AccountTypeEnum.USER, - balance=defaultCredit, + balance=0.0, enabled=True ) - created = self.createAccount(account) - - if defaultCredit > 0: - self.createTransaction(BillingTransaction( - accountId=created["id"], - transactionType=TransactionTypeEnum.CREDIT, - amount=defaultCredit, - description="Initial credit for new user", - referenceType=ReferenceTypeEnum.SYSTEM - )) + self.createAccount(account) existingAccountKeys.add(key) accountsCreated += 1 @@ -662,6 +630,10 @@ class BillingObjects: pagination=pagination, recordFilter=recordFilter ) + _logBillingTransactionsMissingSysCreatedAt( + result["items"], + "getTransactions(accountId) paginated", + ) return PaginatedResult( items=result["items"], totalItems=result["totalItems"], @@ -674,7 +646,7 @@ class BillingObjects: if startDate or endDate: filtered = [] for t in results: - createdAt = t.get("_createdAt") + createdAt = t.get("sysCreatedAt") if createdAt: tDate = createdAt.date() if isinstance(createdAt, datetime) else createdAt if startDate and tDate < startDate: @@ -684,7 +656,7 @@ class BillingObjects: filtered.append(t) results = filtered - results.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) + _sortBillingTransactionsBySysCreatedAtDesc(results, "getTransactions(accountId)") return results[offset:offset + limit] except Exception as e: @@ -739,7 +711,10 @@ class BillingObjects: transactions = self.getTransactions(account["id"], limit=limit) allTransactions.extend(transactions) - allTransactions.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) + _sortBillingTransactionsBySysCreatedAtDesc( + allTransactions, + "getTransactionsByMandate", + ) return allTransactions[:limit] # ========================================================================= @@ -810,35 +785,14 @@ class BillingObjects: """ Check if there's sufficient balance for an operation. - - PREPAY_USER: user.balance >= estimatedCost - - PREPAY_MANDATE: mandate pool balance >= estimatedCost - - User accounts are always ensured to exist (for audit trail). - Root mandate + PREPAY_USER: initial credit from settings.defaultUserCredit on first create. - Missing settings: treated as PREPAY_MANDATE with empty pool (strict). + Checks mandate pool balance against estimatedCost. + User accounts are ensured to exist for audit trail. + Missing settings: treated as PREPAY_MANDATE with empty pool. """ - settings = self.getSettings(mandateId) - if not settings: - billingModel = BillingModelEnum.PREPAY_MANDATE - defaultCredit = 0.0 - else: - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - defaultCredit = float(settings.get("defaultUserCredit", 0.0) or 0.0) + self.getOrCreateUserAccount(mandateId, userId, initialBalance=0.0) - rootMandateId = _getCachedRootMandateId() - isRootMandate = rootMandateId is not None and str(mandateId) == str(rootMandateId) - if billingModel == BillingModelEnum.PREPAY_USER: - initialBalance = defaultCredit if isRootMandate else 0.0 - else: - initialBalance = 0.0 - self.getOrCreateUserAccount(mandateId, userId, initialBalance=initialBalance) - - if billingModel == BillingModelEnum.PREPAY_USER: - account = self.getUserAccount(mandateId, userId) - currentBalance = account.get("balance", 0.0) if account else 0.0 - else: - poolAccount = self.getOrCreateMandateAccount(mandateId) - currentBalance = poolAccount.get("balance", 0.0) + poolAccount = self.getOrCreateMandateAccount(mandateId) + currentBalance = poolAccount.get("balance", 0.0) if currentBalance < estimatedCost: return BillingCheckResult( @@ -846,10 +800,9 @@ class BillingObjects: reason="INSUFFICIENT_BALANCE", currentBalance=currentBalance, requiredAmount=estimatedCost, - billingModel=billingModel, ) - return BillingCheckResult(allowed=True, currentBalance=currentBalance, billingModel=billingModel) + return BillingCheckResult(allowed=True, currentBalance=currentBalance) def recordUsage( self, @@ -870,10 +823,8 @@ class BillingObjects: """ Record usage cost as a billing transaction. - Transaction is ALWAYS recorded on the user's account (clean audit trail). - Balance is deducted from the appropriate account based on billing model: - - PREPAY_USER: deduct from user's own balance - - PREPAY_MANDATE: deduct from mandate pool balance + Transaction is recorded on the user's account (audit trail). + Balance is always deducted from the mandate pool account (PREPAY_MANDATE). """ if priceCHF <= 0: return None @@ -883,9 +834,6 @@ class BillingObjects: logger.debug(f"No billing settings for mandate {mandateId}, skipping usage recording") return None - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - # Transaction is ALWAYS on the user's account (audit trail) userAccount = self.getOrCreateUserAccount(mandateId, userId) transaction = BillingTransaction( @@ -906,18 +854,150 @@ class BillingObjects: errorCount=errorCount ) - # Determine where to deduct balance - if billingModel == BillingModelEnum.PREPAY_USER: - return self.createTransaction(transaction) - if billingModel == BillingModelEnum.PREPAY_MANDATE: - poolAccount = self.getOrCreateMandateAccount(mandateId) - return self.createTransaction(transaction, balanceAccountId=poolAccount["id"]) + poolAccount = self.getOrCreateMandateAccount(mandateId) + return self.createTransaction(transaction, balanceAccountId=poolAccount["id"]) + + def _parseSettingsDateTime(self, value: Any) -> Optional[datetime]: + """Parse datetime from billing settings row (ISO string or datetime).""" + if value is None: + return None + if isinstance(value, datetime): + if value.tzinfo: + return value.astimezone(timezone.utc) + return value.replace(tzinfo=timezone.utc) + if isinstance(value, str): + s = value.replace("Z", "+00:00") + try: + dt = datetime.fromisoformat(s) + except ValueError: + return None + if dt.tzinfo: + return dt.astimezone(timezone.utc) + return dt.replace(tzinfo=timezone.utc) return None - + + def resetStorageBillingPeriod(self, mandateId: str, periodStartAt: datetime) -> None: + """Reset storage watermark state for a new subscription billing period (e.g. Stripe invoice.paid).""" + if periodStartAt.tzinfo is None: + periodStartAt = periodStartAt.replace(tzinfo=timezone.utc) + else: + periodStartAt = periodStartAt.astimezone(timezone.utc) + settings = self.getOrCreateSettings(mandateId) + prev = self._parseSettingsDateTime(settings.get("storagePeriodStartAt")) + if prev is not None and abs((prev - periodStartAt).total_seconds()) < 2: + return + from modules.interfaces.interfaceDbSubscription import _getRootInterface as _getSubRoot + + usedMB = float(_getSubRoot().getMandateDataVolumeMB(mandateId)) + self.updateSettings( + settings["id"], + { + "storageHighWatermarkMB": usedMB, + "storageBilledUpToMB": 0.0, + "storagePeriodStartAt": periodStartAt, + }, + ) + logger.info( + "Storage billing period reset for mandate %s at %s (usedMB=%.2f)", + mandateId, + periodStartAt.isoformat(), + usedMB, + ) + + def reconcileMandateStorageBilling(self, mandateId: str) -> Optional[Dict[str, Any]]: + """Debit prepay pool for new storage overage using period high-watermark (no credit on delete).""" + settings = self.getSettings(mandateId) + if not settings: + return None + from modules.interfaces.interfaceDbSubscription import _getRootInterface as _getSubRoot + from modules.datamodels.datamodelSubscription import _getPlan + + subIface = _getSubRoot() + usedMB = float(subIface.getMandateDataVolumeMB(mandateId)) + sub = subIface.getOperativeForMandate(mandateId) + plan = _getPlan(sub.get("planKey", "")) if sub else None + includedMB = plan.maxDataVolumeMB if plan and plan.maxDataVolumeMB is not None else None + if includedMB is None: + return None + + prevHigh = float(settings.get("storageHighWatermarkMB") or 0.0) + high = max(prevHigh, usedMB) + overageMB = max(0.0, high - float(includedMB)) + billed = float(settings.get("storageBilledUpToMB") or 0.0) + deltaOverage = overageMB - billed + settingsUpdates: Dict[str, Any] = {} + if high != prevHigh: + settingsUpdates["storageHighWatermarkMB"] = high + if deltaOverage <= 1e-9: + if settingsUpdates: + self.updateSettings(settings["id"], settingsUpdates) + return None + + costCHF = round((deltaOverage / 1024.0) * float(STORAGE_PRICE_PER_GB_CHF), 4) + if costCHF <= 0: + if settingsUpdates: + self.updateSettings(settings["id"], settingsUpdates) + return None + + poolAccount = self.getOrCreateMandateAccount(mandateId) + transaction = BillingTransaction( + accountId=poolAccount["id"], + transactionType=TransactionTypeEnum.DEBIT, + amount=costCHF, + description=f"Speicher-Überhang ({deltaOverage:.2f} MB über Plan)", + referenceType=ReferenceTypeEnum.STORAGE, + referenceId=mandateId, + ) + created = self.createTransaction(transaction) + settingsUpdates["storageBilledUpToMB"] = overageMB + self.updateSettings(settings["id"], settingsUpdates) + logger.info( + "Storage overage billed mandate=%s deltaOverageMB=%.4f costCHF=%s", + mandateId, + deltaOverage, + costCHF, + ) + return created + + # ========================================================================= + # Subscription AI-Budget Credit + # ========================================================================= + + def creditSubscriptionBudget(self, mandateId: str, planKey: str, periodLabel: str = "") -> Optional[Dict[str, Any]]: + """Credit the plan's budgetAiCHF to the mandate pool account. + + Should be called once per billing period (initial activation + each invoice.paid). + Returns the created CREDIT transaction or None if budget is 0.""" + from modules.datamodels.datamodelSubscription import _getPlan + + plan = _getPlan(planKey) + if not plan or not plan.budgetAiCHF or plan.budgetAiCHF <= 0: + return None + + poolAccount = self.getOrCreateMandateAccount(mandateId) + description = f"AI-Budget ({planKey})" + if periodLabel: + description += f" – {periodLabel}" + + transaction = BillingTransaction( + accountId=poolAccount["id"], + transactionType=TransactionTypeEnum.CREDIT, + amount=plan.budgetAiCHF, + description=description, + referenceType=ReferenceTypeEnum.SUBSCRIPTION, + referenceId=mandateId, + ) + created = self.createTransaction(transaction) + logger.info( + "AI-Budget credited mandate=%s plan=%s amount=%.2f CHF", + mandateId, planKey, plan.budgetAiCHF, + ) + return created + # ========================================================================= # Workflow Cost Query # ========================================================================= - + def getWorkflowCost(self, workflowId: str) -> float: """Sum of all transaction amounts for a workflow.""" if not workflowId: @@ -928,112 +1008,6 @@ class BillingObjects: ) return sum(t.get("amount", 0.0) for t in transactions) - # ========================================================================= - # Billing Model Switch Operations - # ========================================================================= - - def switchBillingModel(self, mandateId: str, oldModel: BillingModelEnum, newModel: BillingModelEnum) -> Dict[str, Any]: - """ - Switch billing model with budget migration logged as BillingTransactions. - - PREPAY_MANDATE -> PREPAY_USER: pool debited, equal shares credited to user accounts. - PREPAY_USER -> PREPAY_MANDATE: user wallets debited, pool credited with sum. - """ - result = {"oldModel": oldModel.value, "newModel": newModel.value, "migratedAmount": 0.0, "userCount": 0} - - if oldModel == newModel: - return result - - if oldModel == BillingModelEnum.PREPAY_MANDATE and newModel == BillingModelEnum.PREPAY_USER: - poolAccount = self.getMandateAccount(mandateId) - userAccounts = self.db.getRecordset( - BillingAccount, - recordFilter={"mandateId": mandateId, "accountType": AccountTypeEnum.USER.value} - ) - poolBalance = poolAccount.get("balance", 0.0) if poolAccount else 0.0 - n = len(userAccounts) - if poolAccount and poolBalance > 0: - self.createTransaction( - BillingTransaction( - accountId=poolAccount["id"], - transactionType=TransactionTypeEnum.DEBIT, - amount=poolBalance, - description="Model switch: distributed from mandate pool to user wallets", - referenceType=ReferenceTypeEnum.SYSTEM, - ) - ) - result["migratedAmount"] = poolBalance - if n > 0: - remaining = poolBalance - for i, acc in enumerate(userAccounts): - if i == n - 1: - share = round(remaining, 4) - else: - share = round(poolBalance / n, 4) - remaining -= share - if share > 0: - self.createTransaction( - BillingTransaction( - accountId=acc["id"], - transactionType=TransactionTypeEnum.CREDIT, - amount=share, - description="Model switch: share from mandate pool", - referenceType=ReferenceTypeEnum.SYSTEM, - ) - ) - result["userCount"] = n - logger.info( - "Switched %s MANDATE->USER: migrated %.4f CHF to %d user account(s) (transactions logged)", - mandateId, - result["migratedAmount"], - result["userCount"], - ) - return result - - if oldModel == BillingModelEnum.PREPAY_USER and newModel == BillingModelEnum.PREPAY_MANDATE: - userAccounts = self.db.getRecordset( - BillingAccount, - recordFilter={"mandateId": mandateId, "accountType": AccountTypeEnum.USER.value} - ) - totalUserBalance = sum(acc.get("balance", 0.0) for acc in userAccounts) - for acc in userAccounts: - b = acc.get("balance", 0.0) - if b > 0: - self.createTransaction( - BillingTransaction( - accountId=acc["id"], - transactionType=TransactionTypeEnum.DEBIT, - amount=b, - description="Model switch: consolidated to mandate pool", - referenceType=ReferenceTypeEnum.SYSTEM, - ) - ) - poolAccount = self.getOrCreateMandateAccount(mandateId, initialBalance=0.0) - if totalUserBalance > 0: - self.createTransaction( - BillingTransaction( - accountId=poolAccount["id"], - transactionType=TransactionTypeEnum.CREDIT, - amount=totalUserBalance, - description="Model switch: consolidated from user accounts", - referenceType=ReferenceTypeEnum.SYSTEM, - ) - ) - result["migratedAmount"] = totalUserBalance - result["userCount"] = len(userAccounts) - logger.info( - "Switched %s USER->MANDATE: consolidated %.4f CHF from %d users into pool (transactions logged)", - mandateId, - totalUserBalance, - len(userAccounts), - ) - return result - - if newModel == BillingModelEnum.PREPAY_MANDATE: - self.getOrCreateMandateAccount(mandateId, initialBalance=0.0) - - return result - # ========================================================================= # Statistics Operations # ========================================================================= @@ -1128,10 +1102,8 @@ class BillingObjects: def getBalancesForUser(self, userId: str) -> List[BillingBalanceResponse]: """ Get all billing balances for a user across mandates. + Shows the mandate pool balance (shared budget visible to user). - Shows the effective available budget: - - PREPAY_USER: user's own account balance - - PREPAY_MANDATE: mandate pool balance (shared budget visible to user) Args: userId: User ID @@ -1163,27 +1135,15 @@ class BillingObjects: if not settings: continue - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - if billingModel == BillingModelEnum.PREPAY_USER: - account = self.getOrCreateUserAccount(mandateId, userId) - if not account: - continue - balance = account.get("balance", 0.0) - warningThreshold = account.get("warningThreshold", 0.0) - elif billingModel == BillingModelEnum.PREPAY_MANDATE: - poolAccount = self.getOrCreateMandateAccount(mandateId) - if not poolAccount: - continue - balance = poolAccount.get("balance", 0.0) - warningThreshold = poolAccount.get("warningThreshold", 0.0) - else: + poolAccount = self.getOrCreateMandateAccount(mandateId) + if not poolAccount: continue + balance = poolAccount.get("balance", 0.0) + warningThreshold = poolAccount.get("warningThreshold", 0.0) balances.append(BillingBalanceResponse( mandateId=mandateId, mandateName=mandateName, - billingModel=billingModel, balance=balance, warningThreshold=warningThreshold, isWarning=balance <= warningThreshold, @@ -1244,7 +1204,7 @@ class BillingObjects: except Exception as e: logger.error(f"Error getting transactions for user: {e}") - allTransactions.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) + _sortBillingTransactionsBySysCreatedAtDesc(allTransactions, "getTransactionsForUser") return allTransactions[:limit] # ========================================================================= @@ -1280,36 +1240,25 @@ class BillingObjects: if not mandateId: continue - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - # Get mandate info mandate = appInterface.getMandate(mandateId) mandateName = "" if mandate: mandateName = getattr(mandate, 'label', None) or getattr(mandate, 'name', None) or (mandate.get("label") or mandate.get("name", "") if isinstance(mandate, dict) else "") - # Get user accounts count (always exist now for audit trail) - userAccounts = self.db.getRecordset( + allMandateAccounts = self.db.getRecordset( BillingAccount, - recordFilter={"mandateId": mandateId, "accountType": AccountTypeEnum.USER.value} + recordFilter={"mandateId": mandateId} ) - userCount = len(userAccounts) + userCount = sum(1 for acc in allMandateAccounts if acc.get("userId")) - if billingModel == BillingModelEnum.PREPAY_USER: - totalBalance = sum(acc.get("balance", 0.0) for acc in userAccounts) - elif billingModel == BillingModelEnum.PREPAY_MANDATE: - poolAccount = self.getMandateAccount(mandateId) - totalBalance = poolAccount.get("balance", 0.0) if poolAccount else 0.0 - else: - totalBalance = 0.0 + poolAccount = self.getMandateAccount(mandateId) + totalBalance = poolAccount.get("balance", 0.0) if poolAccount else 0.0 balances.append({ "mandateId": mandateId, "mandateName": mandateName, - "billingModel": billingModel.value, "totalBalance": totalBalance, "userCount": userCount, - "defaultUserCredit": float(settings.get("defaultUserCredit", 0.0) or 0.0), "warningThresholdPercent": settings.get("warningThresholdPercent", 10.0), }) @@ -1361,7 +1310,7 @@ class BillingObjects: logger.error(f"Error getting mandate transactions: {e}") # Sort by creation date descending and limit - allTransactions.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) + _sortBillingTransactionsBySysCreatedAtDesc(allTransactions, "getMandateTransactions") return allTransactions[:limit] # ========================================================================= @@ -1385,9 +1334,8 @@ class BillingObjects: try: appInterface = getAppInterface(self.currentUser) - # Get all user accounts - accountFilter = {"accountType": AccountTypeEnum.USER.value} - allAccounts = self.db.getRecordset(BillingAccount, recordFilter=accountFilter) + allAccounts = self.db.getRecordset(BillingAccount) + allAccounts = [acc for acc in allAccounts if acc.get("userId")] # Filter by mandate if specified if mandateIds: @@ -1549,5 +1497,5 @@ class BillingObjects: logger.error(f"Error getting user transactions for mandates: {e}") # Sort by creation date descending and limit - allTransactions.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) + _sortBillingTransactionsBySysCreatedAtDesc(allTransactions, "getUserTransactionsForMandates") return allTransactions[:limit] diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index b0d4aff3..60f4db44 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -251,9 +251,8 @@ class ChatObjects: objectFields[fieldName] = value else: # Field not in model - treat as scalar if simple, otherwise filter out - # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector + # Underscore-prefixed keys (e.g. UI meta) pass through; sys* live on PowerOnModel subclasses if fieldName.startswith("_"): - # Metadata fields should be passed through to connector simpleFields[fieldName] = value elif isinstance(value, (str, int, float, bool, type(None))): simpleFields[fieldName] = value @@ -652,6 +651,32 @@ class ChatObjects: totalPages=totalPages ) + def getLastMessageTimestamp(self, workflowId: str) -> Optional[str]: + """Return the latest publishedAt/sysCreatedAt from ChatMessage for a workflow.""" + messages = self._getRecordset(ChatMessage, recordFilter={"workflowId": workflowId}) + if not messages: + return None + latest = None + for msg in messages: + ts = msg.get("publishedAt") or msg.get("sysCreatedAt") + if ts and (latest is None or str(ts) > str(latest)): + latest = ts + return str(latest) if latest else None + + def searchWorkflowsByContent(self, query: str, limit: int = 50) -> List[str]: + """Return workflow IDs whose messages contain the query string (case-insensitive).""" + allMessages = self._getRecordset(ChatMessage) + matchedIds: set = set() + for msg in allMessages: + content = msg.get("message") or "" + if query in content.lower(): + wfId = msg.get("workflowId") + if wfId: + matchedIds.add(wfId) + if len(matchedIds) >= limit: + break + return list(matchedIds) + def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: """Returns a workflow by ID if user has access.""" # Use RBAC filtering with featureInstanceId for instance-level isolation @@ -885,7 +910,7 @@ class ChatObjects: "role": msg.get("role", "assistant"), "status": msg.get("status", "step"), "sequenceNr": msg.get("sequenceNr", 0), - "publishedAt": msg.get("publishedAt") or msg.get("_createdAt") or msg.get("timestamp") or 0, + "publishedAt": msg.get("publishedAt") or msg.get("sysCreatedAt") or msg.get("timestamp") or 0, "success": msg.get("success"), "actionId": msg.get("actionId"), "actionMethod": msg.get("actionMethod"), @@ -1268,7 +1293,7 @@ class ChatObjects: # CASCADE DELETE: Delete all related data first # 1. Delete message documents (but NOT the files themselves) - # Bypass RBAC -- workflow access already verified, child records may have different _createdBy + # Bypass RBAC -- workflow access already verified, child records may have different sysCreatedBy existing_docs = self.db.getRecordset(ChatDocument, recordFilter={"messageId": messageId}) for doc in existing_docs: self.db.recordDelete(ChatDocument, doc["id"]) @@ -1296,7 +1321,7 @@ class ChatObjects: # Get documents for this message from normalized table - # Bypass RBAC -- workflow access already verified, child records may have different _createdBy + # Bypass RBAC -- workflow access already verified, child records may have different sysCreatedBy documents = self.db.getRecordset(ChatDocument, recordFilter={"messageId": messageId}) if not documents: diff --git a/modules/interfaces/interfaceDbKnowledge.py b/modules/interfaces/interfaceDbKnowledge.py index adf8ed0a..ede37c87 100644 --- a/modules/interfaces/interfaceDbKnowledge.py +++ b/modules/interfaces/interfaceDbKnowledge.py @@ -29,6 +29,7 @@ class KnowledgeObjects: def __init__(self): self.currentUser: Optional[User] = None self.userId: Optional[str] = None + self._scopeCache: Dict[str, List[str]] = {} self._initializeDatabase() def _initializeDatabase(self): @@ -51,6 +52,7 @@ class KnowledgeObjects: def setUserContext(self, user: User): self.currentUser = user self.userId = user.id if user else None + self._scopeCache = {} if self.userId: self.db.updateContext(self.userId) @@ -89,10 +91,20 @@ class KnowledgeObjects: def deleteFileContentIndex(self, fileId: str) -> bool: """Delete a FileContentIndex and all associated ContentChunks.""" + existing = self.getFileContentIndex(fileId) + mandateId = (existing or {}).get("mandateId") or "" chunks = self.db.getRecordset(ContentChunk, recordFilter={"fileId": fileId}) for chunk in chunks: self.db.recordDelete(ContentChunk, chunk["id"]) - return self.db.recordDelete(FileContentIndex, fileId) + ok = self.db.recordDelete(FileContentIndex, fileId) + if ok and mandateId: + try: + from modules.interfaces.interfaceDbBilling import _getRootInterface + + _getRootInterface().reconcileMandateStorageBilling(str(mandateId)) + except Exception as ex: + logger.warning("reconcileMandateStorageBilling after delete failed: %s", ex) + return ok # ========================================================================= # ContentChunk CRUD @@ -215,25 +227,88 @@ class KnowledgeObjects: # Semantic Search # ========================================================================= + def _buildScopeFilter(self, userId: str = None, featureInstanceId: str = None, mandateId: str = None) -> dict: + """Build a scope-aware filter for RAG queries. + Returns a filter dict that includes records visible to this user context.""" + return { + "userId": userId, + "featureInstanceId": featureInstanceId, + "mandateId": mandateId, + } + + def _getScopedFileIds(self, userId: str = None, featureInstanceId: str = None, mandateId: str = None, isSysAdmin: bool = False) -> List[str]: + """Collect FileContentIndex IDs visible under the scope union: + - scope=personal AND userId matches + - scope=featureInstance AND featureInstanceId matches + - scope=mandate AND mandateId matches + - scope=global (only if isSysAdmin) + """ + _cacheKey = f"{userId}:{featureInstanceId}:{mandateId}:{isSysAdmin}" + if _cacheKey in self._scopeCache: + return self._scopeCache[_cacheKey] + + allIds: set = set() + + if isSysAdmin: + globalIndexes = self.db.getRecordset( + FileContentIndex, recordFilter={"scope": "global"} + ) + for idx in globalIndexes: + fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if fid: + allIds.add(fid) + + if userId: + personalIndexes = self.db.getRecordset( + FileContentIndex, recordFilter={"scope": "personal", "userId": userId} + ) + for idx in personalIndexes: + fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if fid: + allIds.add(fid) + + if featureInstanceId: + instanceIndexes = self.db.getRecordset( + FileContentIndex, recordFilter={"scope": "featureInstance", "featureInstanceId": featureInstanceId} + ) + for idx in instanceIndexes: + fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if fid: + allIds.add(fid) + + if mandateId: + mandateIndexes = self.db.getRecordset( + FileContentIndex, recordFilter={"scope": "mandate", "mandateId": mandateId} + ) + for idx in mandateIndexes: + fid = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if fid: + allIds.add(fid) + + self._scopeCache[_cacheKey] = list(allIds) + return self._scopeCache[_cacheKey] + def semanticSearch( self, queryVector: List[float], userId: str = None, featureInstanceId: str = None, mandateId: str = None, - isShared: bool = None, + scope: str = None, limit: int = 10, minScore: float = None, contentType: str = None, + isSysAdmin: bool = False, ) -> List[Dict[str, Any]]: """Semantic search across ContentChunks using pgvector cosine similarity. Args: queryVector: Query embedding vector. - userId: Filter by user (Instance Layer). + userId: Filter by user (personal scope). featureInstanceId: Filter by feature instance. - mandateId: Filter by mandate (for Shared Layer lookups). - isShared: If True, search Shared Layer via FileContentIndex join. + mandateId: Filter by mandate (scope=mandate means visible to all mandate users). + scope: If provided, filter by this specific scope value. + If not provided, use scope-union approach (personal + featureInstance + mandate + global). limit: Max results. minScore: Minimum cosine similarity (0.0 - 1.0). contentType: Filter by content type (text, image, etc.). @@ -242,25 +317,36 @@ class KnowledgeObjects: List of ContentChunk records with _score field, sorted by relevance. """ recordFilter = {} - if userId: - recordFilter["userId"] = userId - if featureInstanceId: - recordFilter["featureInstanceId"] = featureInstanceId if contentType: recordFilter["contentType"] = contentType - if isShared and mandateId: - sharedIndexes = self.db.getRecordset( - FileContentIndex, - recordFilter={"mandateId": mandateId, "isShared": True}, + if scope: + scopeFilter: Dict[str, Any] = {"scope": scope} + if mandateId: + scopeFilter["mandateId"] = mandateId + if featureInstanceId: + scopeFilter["featureInstanceId"] = featureInstanceId + scopedFileIds = self.db.getRecordset( + FileContentIndex, recordFilter=scopeFilter ) - sharedFileIds = [idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) for idx in sharedIndexes] - sharedFileIds = [fid for fid in sharedFileIds if fid] - if not sharedFileIds: + fileIds = [ + idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + for idx in scopedFileIds + ] + fileIds = [fid for fid in fileIds if fid] + if not fileIds: return [] - recordFilter.pop("userId", None) - recordFilter.pop("featureInstanceId", None) - recordFilter["fileId"] = sharedFileIds + recordFilter["fileId"] = fileIds + elif userId or featureInstanceId or mandateId: + scopedFileIds = self._getScopedFileIds( + userId=userId, + featureInstanceId=featureInstanceId, + mandateId=mandateId, + isSysAdmin=isSysAdmin, + ) + if not scopedFileIds: + return [] + recordFilter["fileId"] = scopedFileIds return self.db.semanticSearch( modelClass=ContentChunk, @@ -317,7 +403,7 @@ class KnowledgeObjects: if mandateId: files_shared = self.db.getRecordset( FileContentIndex, - recordFilter={"mandateId": mandateId, "isShared": True}, + recordFilter={"mandateId": mandateId, "scope": "mandate"}, ) by_id: Dict[str, Dict[str, Any]] = {} @@ -466,6 +552,76 @@ class KnowledgeObjects: } +def aggregateMandateRagTotalBytes(mandateId: str) -> int: + """Sum FileContentIndex.totalSize for a mandate. + + Primary strategy (relies on correct scope fields on FileContentIndex): + 1. FileContentIndex rows with mandateId on the index + 2. FileContentIndex rows with featureInstanceId of any mandate FeatureInstance + Deduplicates by id. + """ + if not mandateId: + return 0 + from modules.datamodels.datamodelFeatures import FeatureInstance + from modules.interfaces.interfaceDbApp import getRootInterface + + knowDb = getInterface(None).db + appDb = getRootInterface().db + byId: Dict[str, Dict[str, Any]] = {} + + for row in knowDb.getRecordset(FileContentIndex, recordFilter={"mandateId": mandateId}): + rid = row.get("id") + if rid: + byId[str(rid)] = row + + instances = appDb.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) + instIds = [str(inst.get("id", "")) for inst in instances if inst.get("id")] + + for instId in instIds: + for row in knowDb.getRecordset(FileContentIndex, recordFilter={"featureInstanceId": instId}): + rid = row.get("id") + if rid and str(rid) not in byId: + byId[str(rid)] = row + + # DEPRECATED: file-ID-correlation fallback from poweron_management. + # Only needed for pre-migration data where mandateId/featureInstanceId on the + # FileContentIndex are empty. Remove once migrateRagScopeFields has been run. + _fallbackCount = 0 + try: + from modules.datamodels.datamodelFiles import FileItem + from modules.interfaces.interfaceDbManagement import ComponentObjects + mgmtDb = ComponentObjects().db + knowledgeIf = getInterface(None) + + fileIds: set = set() + for f in mgmtDb.getRecordset(FileItem, recordFilter={"mandateId": mandateId}): + fid = f.get("id") if isinstance(f, dict) else getattr(f, "id", None) + if fid: + fileIds.add(str(fid)) + for instId in instIds: + for f in mgmtDb.getRecordset(FileItem, recordFilter={"featureInstanceId": instId}): + fid = f.get("id") if isinstance(f, dict) else getattr(f, "id", None) + if fid: + fileIds.add(str(fid)) + + for fid in fileIds: + if fid in byId: + continue + row = knowledgeIf.getFileContentIndex(fid) + if row: + byId[fid] = row + _fallbackCount += 1 + except Exception as e: + logger.warning("aggregateMandateRagTotalBytes fallback failed: %s", e) + + total = sum(int(r.get("totalSize") or 0) for r in byId.values()) + logger.info( + "aggregateMandateRagTotalBytes(%s): %d indexes, %d bytes (fallback: %d)", + mandateId, len(byId), total, _fallbackCount, + ) + return total + + def getInterface(currentUser: Optional[User] = None) -> KnowledgeObjects: """Get or create a KnowledgeObjects singleton.""" if "default" not in _instances: diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 64883b95..2df85164 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -175,12 +175,7 @@ class ComponentObjects: # Complex objects that should be filtered out objectFields[fieldName] = value else: - # Field not in model - treat as scalar if simple, otherwise filter out - # BUT: always include metadata fields (_createdBy, _createdAt, etc.) as they're handled by connector - if fieldName.startswith("_"): - # Metadata fields should be passed through to connector - simpleFields[fieldName] = value - elif isinstance(value, (str, int, float, bool, type(None))): + if isinstance(value, (str, int, float, bool, type(None))): simpleFields[fieldName] = value else: objectFields[fieldName] = value @@ -609,7 +604,7 @@ class ComponentObjects: """ isSysAdmin = self._isSysAdmin() for prompt in prompts: - isOwner = prompt.get("_createdBy") == self.userId + isOwner = prompt.get("sysCreatedBy") == self.userId prompt["_permissions"] = { "canUpdate": isOwner or isSysAdmin, "canDelete": isOwner or isSysAdmin @@ -621,13 +616,13 @@ class ComponentObjects: Visibility rules: - SysAdmin: ALL prompts - - Regular user: own prompts (_createdBy) + system prompts (isSystem=True) + - Regular user: own prompts (sysCreatedBy) + system prompts (isSystem=True) """ if self._isSysAdmin(): return self.db.getRecordset(Prompt) # Get own prompts - ownPrompts = self.db.getRecordset(Prompt, recordFilter={"_createdBy": self.userId}) + ownPrompts = self.db.getRecordset(Prompt, recordFilter={"sysCreatedBy": self.userId}) # Get system prompts systemPrompts = self.db.getRecordset(Prompt, recordFilter={"isSystem": True}) @@ -716,7 +711,7 @@ class ComponentObjects: # Visibility check for non-SysAdmin: must be owner or system prompt if not self._isSysAdmin(): - isOwner = prompt.get("_createdBy") == self.userId + isOwner = prompt.get("sysCreatedBy") == self.userId isSystem = prompt.get("isSystem", False) if not isOwner and not isSystem: return None @@ -747,7 +742,7 @@ class ComponentObjects: raise ValueError(f"Prompt {promptId} not found") # Permission check: owner or SysAdmin - isOwner = (getattr(prompt, '_createdBy', None) == self.userId) + isOwner = (getattr(prompt, 'sysCreatedBy', None) == self.userId) if not self._isSysAdmin() and not isOwner: raise PermissionError(f"No permission to update prompt {promptId}") @@ -784,7 +779,7 @@ class ComponentObjects: return False # Permission check: owner or SysAdmin - isOwner = (getattr(prompt, '_createdBy', None) == self.userId) + isOwner = (getattr(prompt, 'sysCreatedBy', None) == self.userId) if not self._isSysAdmin() and not isOwner: raise PermissionError(f"No permission to delete prompt {promptId}") @@ -798,7 +793,7 @@ class ComponentObjects: def checkForDuplicateFile(self, fileHash: str, fileName: str) -> Optional[FileItem]: """Checks if a file with the same hash AND fileName already exists for the current user. - Duplicate = same user (_createdBy) + same fileHash + same fileName. + Duplicate = same user (sysCreatedBy) + same fileHash + same fileName. Same hash with different name is allowed (intentional copy by user). Uses direct DB query (not RBAC) because files are isolated per user. """ @@ -809,7 +804,7 @@ class ComponentObjects: matchingFiles = self.db.getRecordset( FileItem, recordFilter={ - "_createdBy": self.userId, + "sysCreatedBy": self.userId, "fileHash": fileHash, "fileName": fileName } @@ -828,7 +823,7 @@ class ComponentObjects: mimeType=file["mimeType"], fileHash=file["fileHash"], fileSize=file["fileSize"], - creationDate=file["creationDate"] + sysCreatedAt=file.get("sysCreatedAt") or file.get("creationDate"), ) def getMimeType(self, fileName: str) -> str: @@ -908,7 +903,7 @@ class ComponentObjects: def _getFilesByCurrentUser(self, recordFilter: Dict[str, Any] = None) -> List[Dict[str, Any]]: """Files are always user-scoped. Returns only files owned by the current user, regardless of role (including SysAdmin). This bypasses RBAC intentionally.""" - filterDict = {"_createdBy": self.userId} + filterDict = {"sysCreatedBy": self.userId} if recordFilter: filterDict.update(recordFilter) return self.db.getRecordset(FileItem, recordFilter=filterDict) @@ -927,20 +922,27 @@ class ComponentObjects: If pagination is provided: PaginatedResult with items and metadata """ # User-scoping filter: every user only sees their own files (bypasses RBAC SysAdmin override) - recordFilter = {"_createdBy": self.userId} + recordFilter = {"sysCreatedBy": self.userId} def _convertFileItems(files): fileItems = [] for file in files: try: - creationDate = file.get("creationDate") - if creationDate is None or not isinstance(creationDate, (int, float)) or creationDate <= 0: - file["creationDate"] = getUtcTimestamp() + sysCreatedAt = file.get("sysCreatedAt") or file.get("creationDate") + if sysCreatedAt is None or not isinstance(sysCreatedAt, (int, float)) or sysCreatedAt <= 0: + file["sysCreatedAt"] = getUtcTimestamp() + else: + file["sysCreatedAt"] = sysCreatedAt fileName = file.get("fileName") if not fileName or fileName == "None": continue - + + if file.get("scope") is None: + file["scope"] = "personal" + if file.get("neutralize") is None: + file["neutralize"] = False + fileItem = FileItem(**file) fileItems.append(fileItem) except Exception as e: @@ -969,7 +971,7 @@ class ComponentObjects: def getFile(self, fileId: str) -> Optional[FileItem]: """Returns a file by ID if it belongs to the current user (user-scoped).""" - # Files are always user-scoped: filter by _createdBy (bypasses RBAC SysAdmin override) + # Files are always user-scoped: filter by sysCreatedBy (bypasses RBAC SysAdmin override) filteredFiles = self._getFilesByCurrentUser(recordFilter={"id": fileId}) if not filteredFiles: @@ -977,20 +979,19 @@ class ComponentObjects: file = filteredFiles[0] try: - # Get creation date from record or use current time - creationDate = file.get("creationDate") - if not creationDate: - creationDate = getUtcTimestamp() - + sysCreatedAt = file.get("sysCreatedAt") or file.get("creationDate") + if not sysCreatedAt: + sysCreatedAt = getUtcTimestamp() + return FileItem( id=file.get("id"), mandateId=file.get("mandateId"), + featureInstanceId=file.get("featureInstanceId", ""), fileName=file.get("fileName"), mimeType=file.get("mimeType"), - workflowId=file.get("workflowId"), fileHash=file.get("fileHash"), fileSize=file.get("fileSize"), - creationDate=creationDate + sysCreatedAt=sysCreatedAt, ) except Exception as e: logger.error(f"Error converting file record: {str(e)}") @@ -1053,15 +1054,20 @@ class ComponentObjects: # Ensure fileName is unique uniqueName = self._generateUniquefileName(name) - # Use mandateId and featureInstanceId from context for proper data isolation - # Convert None to empty string to satisfy Pydantic validation mandateId = self.mandateId or "" featureInstanceId = self.featureInstanceId or "" - - # Create FileItem instance + + if featureInstanceId: + scope = "featureInstance" + elif mandateId: + scope = "mandate" + else: + scope = "personal" + fileItem = FileItem( mandateId=mandateId, featureInstanceId=featureInstanceId, + scope=scope, fileName=uniqueName, mimeType=mimeType, fileSize=fileSize, @@ -1146,7 +1152,7 @@ class ComponentObjects: self.db._ensure_connection() with self.db.connection.cursor() as cursor: cursor.execute( - 'SELECT "id" FROM "FileItem" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'SELECT "id" FROM "FileItem" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (uniqueIds, self.userId or ""), ) accessibleIds = [row["id"] for row in cursor.fetchall()] @@ -1157,7 +1163,7 @@ class ComponentObjects: cursor.execute('DELETE FROM "FileData" WHERE "id" = ANY(%s)', (accessibleIds,)) cursor.execute( - 'DELETE FROM "FileItem" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'DELETE FROM "FileItem" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (accessibleIds, self.userId or ""), ) deletedFiles = cursor.rowcount @@ -1202,12 +1208,12 @@ class ComponentObjects: def getFolder(self, folderId: str) -> Optional[Dict[str, Any]]: """Returns a folder by ID if it belongs to the current user.""" - folders = self.db.getRecordset(FileFolder, recordFilter={"id": folderId, "_createdBy": self.userId or ""}) + folders = self.db.getRecordset(FileFolder, recordFilter={"id": folderId, "sysCreatedBy": self.userId or ""}) return folders[0] if folders else None def listFolders(self, parentId: Optional[str] = None) -> List[Dict[str, Any]]: """List folders for current user, optionally filtered by parentId.""" - recordFilter = {"_createdBy": self.userId or ""} + recordFilter = {"sysCreatedBy": self.userId or ""} if parentId is not None: recordFilter["parentId"] = parentId return self.db.getRecordset(FileFolder, recordFilter=recordFilter) @@ -1256,7 +1262,7 @@ class ComponentObjects: self.db._ensure_connection() with self.db.connection.cursor() as cursor: cursor.execute( - 'SELECT "id" FROM "FileItem" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'SELECT "id" FROM "FileItem" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (uniqueIds, self.userId or ""), ) accessibleIds = [row["id"] for row in cursor.fetchall()] @@ -1265,8 +1271,8 @@ class ComponentObjects: raise FileNotFoundError(f"Files not found or not accessible: {missingIds}") cursor.execute( - 'UPDATE "FileItem" SET "folderId" = %s, "_modifiedAt" = %s, "_modifiedBy" = %s ' - 'WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'UPDATE "FileItem" SET "folderId" = %s, "sysModifiedAt" = %s, "sysModifiedBy" = %s ' + 'WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (targetFolderId, getUtcTimestamp(), self.userId or "", accessibleIds, self.userId or ""), ) movedFiles = cursor.rowcount @@ -1295,7 +1301,7 @@ class ComponentObjects: existingInTarget = self.db.getRecordset( FileFolder, - recordFilter={"parentId": targetParentId or "", "_createdBy": self.userId or ""}, + recordFilter={"parentId": targetParentId or "", "sysCreatedBy": self.userId or ""}, ) existingNames = {f.get("name"): f.get("id") for f in existingInTarget} movingNames: Dict[str, str] = {} @@ -1316,8 +1322,8 @@ class ComponentObjects: self.db._ensure_connection() with self.db.connection.cursor() as cursor: cursor.execute( - 'UPDATE "FileFolder" SET "parentId" = %s, "_modifiedAt" = %s, "_modifiedBy" = %s ' - 'WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'UPDATE "FileFolder" SET "parentId" = %s, "sysModifiedAt" = %s, "sysModifiedBy" = %s ' + 'WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (targetParentId, getUtcTimestamp(), self.userId or "", uniqueIds, self.userId or ""), ) movedFolders = cursor.rowcount @@ -1335,7 +1341,7 @@ class ComponentObjects: if not folder: raise FileNotFoundError(f"Folder {folderId} not found") - childFolders = self.db.getRecordset(FileFolder, recordFilter={"parentId": folderId, "_createdBy": self.userId or ""}) + childFolders = self.db.getRecordset(FileFolder, recordFilter={"parentId": folderId, "sysCreatedBy": self.userId or ""}) childFiles = self._getFilesByCurrentUser(recordFilter={"folderId": folderId}) if not recursive and (childFolders or childFiles): @@ -1384,7 +1390,7 @@ class ComponentObjects: self.db._ensure_connection() with self.db.connection.cursor() as cursor: cursor.execute( - 'SELECT "id" FROM "FileFolder" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'SELECT "id" FROM "FileFolder" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (uniqueIds, self.userId or ""), ) rootAccessibleIds = [row["id"] for row in cursor.fetchall()] @@ -1397,12 +1403,12 @@ class ComponentObjects: WITH RECURSIVE folder_tree AS ( SELECT "id" FROM "FileFolder" - WHERE "id" = ANY(%s) AND "_createdBy" = %s + WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s UNION ALL SELECT child."id" FROM "FileFolder" child INNER JOIN folder_tree ft ON child."parentId" = ft."id" - WHERE child."_createdBy" = %s + WHERE child."sysCreatedBy" = %s ) SELECT DISTINCT "id" FROM folder_tree """, @@ -1411,7 +1417,7 @@ class ComponentObjects: allFolderIds = [row["id"] for row in cursor.fetchall()] cursor.execute( - 'SELECT "id" FROM "FileItem" WHERE "folderId" = ANY(%s) AND "_createdBy" = %s', + 'SELECT "id" FROM "FileItem" WHERE "folderId" = ANY(%s) AND "sysCreatedBy" = %s', (allFolderIds, self.userId or ""), ) allFileIds = [row["id"] for row in cursor.fetchall()] @@ -1419,7 +1425,7 @@ class ComponentObjects: if allFileIds: cursor.execute('DELETE FROM "FileData" WHERE "id" = ANY(%s)', (allFileIds,)) cursor.execute( - 'DELETE FROM "FileItem" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'DELETE FROM "FileItem" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (allFileIds, self.userId or ""), ) deletedFiles = cursor.rowcount @@ -1427,7 +1433,7 @@ class ComponentObjects: deletedFiles = 0 cursor.execute( - 'DELETE FROM "FileFolder" WHERE "id" = ANY(%s) AND "_createdBy" = %s', + 'DELETE FROM "FileFolder" WHERE "id" = ANY(%s) AND "sysCreatedBy" = %s', (allFolderIds, self.userId or ""), ) deletedFolders = cursor.rowcount diff --git a/modules/interfaces/interfaceDbSubscription.py b/modules/interfaces/interfaceDbSubscription.py index f08025ea..f1d7ccf7 100644 --- a/modules/interfaces/interfaceDbSubscription.py +++ b/modules/interfaces/interfaceDbSubscription.py @@ -293,9 +293,45 @@ class SubscriptionObjects: if current + delta > cap: from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionCapacityException raise SubscriptionCapacityException(resourceType=resourceType, currentCount=current, maxAllowed=cap) + elif resourceType == "dataVolumeMB": + cap = plan.maxDataVolumeMB + if cap is None: + return True + currentMB = self.getMandateDataVolumeMB(mandateId) + if currentMB + delta > cap: + from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionCapacityException + raise SubscriptionCapacityException(resourceType=resourceType, currentCount=int(currentMB), maxAllowed=cap) return True + def getMandateDataVolumeMB(self, mandateId: str) -> float: + """Total indexed data volume for the mandate (MB), for billing and capacity checks.""" + return self._getMandateDataVolumeMB(mandateId) + + def _getMandateDataVolumeMB(self, mandateId: str) -> float: + """Sum RAG index size (FileContentIndex.totalSize) for the mandate; reads poweron_knowledge.""" + try: + from modules.interfaces.interfaceDbKnowledge import aggregateMandateRagTotalBytes + + return aggregateMandateRagTotalBytes(mandateId) / (1024 * 1024) + except Exception: + return 0.0 + + def getDataVolumeWarning(self, mandateId: str) -> Optional[Dict[str, Any]]: + """Return a warning dict if mandate uses >=80% of maxDataVolumeMB, else None.""" + sub = self.getOperativeForMandate(mandateId) + if not sub: + return None + plan = self.getPlan(sub.get("planKey", "")) + if not plan or not plan.maxDataVolumeMB: + return None + usedMB = self.getMandateDataVolumeMB(mandateId) + limitMB = plan.maxDataVolumeMB + percent = (usedMB / limitMB * 100) if limitMB > 0 else 0 + if percent >= 80: + return {"usedMB": round(usedMB, 2), "limitMB": limitMB, "percent": round(percent, 1), "warning": True} + return {"usedMB": round(usedMB, 2), "limitMB": limitMB, "percent": round(percent, 1), "warning": False} + # ========================================================================= # Counting (cross-DB queries against poweron_app) # ========================================================================= @@ -321,11 +357,18 @@ class SubscriptionObjects: # Stripe quantity sync # ========================================================================= - def syncQuantityToStripe(self, subscriptionId: str) -> None: + def syncQuantityToStripe(self, subscriptionId: str, *, raiseOnError: bool = False) -> None: """Update Stripe subscription item quantities to match actual active counts. - Takes subscriptionId, not mandateId.""" + Takes subscriptionId, not mandateId. + + Args: + raiseOnError: If True, propagate Stripe API errors instead of logging them. + Use True for billing-critical paths (store activation). + """ sub = self.getById(subscriptionId) if not sub or not sub.get("stripeSubscriptionId"): + if raiseOnError: + raise ValueError(f"Subscription {subscriptionId} hat keine Stripe-Anbindung — Abrechnung nicht möglich.") return mandateId = sub["mandateId"] @@ -351,3 +394,5 @@ class SubscriptionObjects: logger.info("Stripe quantity synced for sub %s: users=%d, instances=%d", subscriptionId, activeUsers, activeInstances) except Exception as e: logger.error("syncQuantityToStripe(%s) failed: %s", subscriptionId, e) + if raiseOnError: + raise diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py index b261e76e..6616218d 100644 --- a/modules/interfaces/interfaceFeatures.py +++ b/modules/interfaces/interfaceFeatures.py @@ -57,7 +57,7 @@ class FeatureInterface: records = self.db.getRecordset(Feature, recordFilter={"code": featureCode}) if not records: return None - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return Feature(**cleanedRecord) except Exception as e: logger.error(f"Error getting feature {featureCode}: {e}") @@ -74,7 +74,7 @@ class FeatureInterface: records = self.db.getRecordset(Feature) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Feature(**cleanedRecord)) return result except Exception as e: @@ -120,7 +120,7 @@ class FeatureInterface: records = self.db.getRecordset(FeatureInstance, recordFilter={"id": instanceId}) if not records: return None - cleanedRecord = {k: v for k, v in records[0].items() if not k.startswith("_")} + cleanedRecord = dict(records[0]) return FeatureInstance(**cleanedRecord) except Exception as e: logger.error(f"Error getting feature instance {instanceId}: {e}") @@ -144,7 +144,7 @@ class FeatureInterface: records = self.db.getRecordset(FeatureInstance, recordFilter=recordFilter) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(FeatureInstance(**cleanedRecord)) return result except Exception as e: @@ -199,7 +199,7 @@ class FeatureInterface: if copyTemplateRoles: self._copyTemplateRoles(featureCode, mandateId, instanceId) - cleanedRecord = {k: v for k, v in createdInstance.items() if not k.startswith("_")} + cleanedRecord = dict(createdInstance) return FeatureInstance(**cleanedRecord) except Exception as e: @@ -208,7 +208,11 @@ class FeatureInterface: def _copyTemplateRoles(self, featureCode: str, mandateId: str, instanceId: str) -> int: """ - Copy global template roles for a feature to a new instance. + Copy feature-specific template roles to a new instance. + + INVARIANT: Feature instances MUST receive feature-specific roles + (e.g. workspace-admin, workspace-user). NEVER generic mandate roles. + Feature templates have featureCode set and isSystemRole=False. Args: featureCode: Feature code @@ -217,19 +221,30 @@ class FeatureInterface: Returns: Number of roles copied + + Raises: + ValueError: If no feature-specific template roles exist """ try: - # Find global template roles for this feature (mandateId=None) - globalRoles = self.db.getRecordset( + allTemplates = self.db.getRecordset( Role, - recordFilter={"featureCode": featureCode, "mandateId": None} + recordFilter={"featureCode": featureCode} ) - if not globalRoles: - logger.debug(f"No template roles found for feature {featureCode}") - return 0 + featureTemplates = [ + r for r in allTemplates + if r.get("mandateId") is None and r.get("featureInstanceId") is None + ] - templateRoleIds = [r.get("id") for r in globalRoles] + if not featureTemplates: + raise ValueError( + f"No feature-specific template roles found for '{featureCode}'. " + f"Each feature module must define TEMPLATE_ROLES and sync them to DB on startup." + ) + + logger.info(f"Found {len(featureTemplates)} feature-specific template roles for '{featureCode}'") + + templateRoleIds = [r.get("id") for r in featureTemplates] # BULK: Load all template AccessRules in one query allTemplateRules = [] @@ -246,7 +261,7 @@ class FeatureInterface: # Copy roles and their AccessRules copiedCount = 0 - for templateRole in globalRoles: + for templateRole in featureTemplates: newRoleId = str(uuid.uuid4()) # Create new role for this instance @@ -282,9 +297,11 @@ class FeatureInterface: logger.info(f"Copied {copiedCount} template roles for instance {instanceId}") return copiedCount + except ValueError: + raise except Exception as e: logger.error(f"Error copying template roles: {e}") - return 0 + raise ValueError(f"Failed to copy template roles for '{featureCode}': {e}") def syncRolesFromTemplate(self, featureInstanceId: str, addOnly: bool = True) -> Dict[str, int]: """ @@ -309,11 +326,15 @@ class FeatureInterface: featureCode = instance.featureCode mandateId = instance.mandateId - # Get current template roles - templateRoles = self.db.getRecordset( + # Get feature-specific template roles (mandateId=None, featureInstanceId=None) + allForFeature = self.db.getRecordset( Role, - recordFilter={"featureCode": featureCode, "mandateId": None} + recordFilter={"featureCode": featureCode} ) + templateRoles = [ + r for r in allForFeature + if r.get("mandateId") is None and r.get("featureInstanceId") is None + ] templateLabels = {r.get("roleLabel") for r in templateRoles} # Get current instance roles @@ -414,7 +435,7 @@ class FeatureInterface: updated = self.db.recordModify(FeatureInstance, instanceId, filteredData) if updated: - cleanedRecord = {k: v for k, v in updated.items() if not k.startswith("_")} + cleanedRecord = dict(updated) return FeatureInstance(**cleanedRecord) return None except Exception as e: @@ -463,7 +484,7 @@ class FeatureInterface: records = self.db.getRecordset(Role, recordFilter=recordFilter) result = [] for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_")} + cleanedRecord = dict(record) result.append(Role(**cleanedRecord)) return result except Exception as e: diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index e65cd5ab..947a6e2d 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -17,7 +17,7 @@ Data Namespace Structure: GROUP-Berechtigung: - data.uam.*: GROUP filtert nach Mandant (via UserMandate) -- data.chat.*, data.files.*, data.automation.*: GROUP = MY (benutzer-eigen); bei gesetztem featureInstanceId zusätzlich _createdBy +- data.chat.*, data.files.*, data.automation.*: GROUP = MY (benutzer-eigen); bei gesetztem featureInstanceId zusätzlich sysCreatedBy - data.feature.*: GROUP filtert nach mandateId/featureInstanceId """ @@ -146,7 +146,7 @@ def getRecordsetWithRBAC( mandateId: Explicit mandate context (from request header). Required for GROUP access. featureInstanceId: Explicit feature instance context enrichPermissions: If True, adds _permissions field to each record with row-level - permissions { canUpdate, canDelete } based on RBAC rules and _createdBy + permissions { canUpdate, canDelete } based on RBAC rules and sysCreatedBy featureCode: Optional feature code for feature-specific tables (e.g., "trustee"). If None, table is treated as a system table. @@ -657,7 +657,7 @@ def buildRbacWhereClause( # shared featureInstance (stale RBAC rules or merged roles). Same as MY. namespaceAll = TABLE_NAMESPACE.get(table, "system") if featureInstanceId and namespaceAll == "chat": - userIdFieldAll = "_createdBy" + userIdFieldAll = "sysCreatedBy" if table == "UserInDB": userIdFieldAll = "id" elif table == "UserConnection": @@ -671,7 +671,7 @@ def buildRbacWhereClause( return {"condition": " AND ".join(baseConditions), "values": baseValues} return None - # My records - filter by _createdBy or userId field + # My records - filter by sysCreatedBy or userId field if readLevel == AccessLevel.MY: # Try common field names for creator userIdField = None @@ -680,7 +680,7 @@ def buildRbacWhereClause( elif table == "UserConnection": userIdField = "userId" else: - userIdField = "_createdBy" + userIdField = "sysCreatedBy" conditions = list(baseConditions) values = list(baseValues) @@ -707,7 +707,7 @@ def buildRbacWhereClause( if featureInstanceId and readLevel == AccessLevel.GROUP: conditions = list(baseConditions) values = list(baseValues) - conditions.append('"_createdBy" = %s') + conditions.append('"sysCreatedBy" = %s') values.append(currentUser.id) return {"condition": " AND ".join(conditions), "values": values} return {"condition": " AND ".join(baseConditions), "values": baseValues} @@ -829,7 +829,7 @@ def _enrichRecordsWithPermissions( Logic: - AccessLevel.ALL ('a'): User can update/delete all records - - AccessLevel.MY ('m'): User can only update/delete records where _createdBy == userId + - AccessLevel.MY ('m'): User can only update/delete records where sysCreatedBy == userId - AccessLevel.GROUP ('g'): Same as MY for now (group-level ownership) - AccessLevel.NONE ('n'): User cannot update/delete any records @@ -846,7 +846,7 @@ def _enrichRecordsWithPermissions( for record in records: recordCopy = dict(record) - createdBy = record.get("_createdBy") + createdBy = record.get("sysCreatedBy") # Determine canUpdate canUpdate = _checkRowPermission(permissions.update, userId, createdBy) @@ -873,7 +873,7 @@ def _checkRowPermission( Args: accessLevel: The permission level (ALL, MY, GROUP, NONE) userId: Current user's ID - recordCreatedBy: The _createdBy value of the record + recordCreatedBy: The sysCreatedBy value of the record Returns: True if user has permission, False otherwise @@ -884,9 +884,9 @@ def _checkRowPermission( if accessLevel == AccessLevel.ALL: return True - # MY and GROUP: Check ownership via _createdBy + # MY and GROUP: Check ownership via sysCreatedBy if accessLevel in (AccessLevel.MY, AccessLevel.GROUP): - # If record has no _createdBy, allow access (can't verify ownership) + # If record has no sysCreatedBy, allow access (can't verify ownership) if not recordCreatedBy: return True # If no userId, can't verify - deny diff --git a/modules/interfaces/interfaceVoiceObjects.py b/modules/interfaces/interfaceVoiceObjects.py index dc391bae..38807bac 100644 --- a/modules/interfaces/interfaceVoiceObjects.py +++ b/modules/interfaces/interfaceVoiceObjects.py @@ -11,9 +11,7 @@ import logging from typing import AsyncGenerator, Callable, Dict, Any, Optional, List from modules.connectors.connectorVoiceGoogle import ConnectorGoogleSpeech -from modules.datamodels.datamodelVoice import VoiceSettings from modules.datamodels.datamodelUam import User -from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) @@ -335,123 +333,6 @@ class VoiceObjects: "error": str(e) } - # Voice Settings Management - - def getVoiceSettings(self, userId: str) -> Optional[VoiceSettings]: - """ - Get voice settings for a user. - - Args: - userId: User ID to get settings for - - Returns: - VoiceSettings object or None if not found - """ - try: - # This would typically query the database - # For now, return None as this is handled by the database interface - logger.debug(f"Getting voice settings for user: {userId}") - return None - - except Exception as e: - logger.error(f"❌ Error getting voice settings: {e}") - return None - - def createVoiceSettings(self, settingsData: Dict[str, Any]) -> Optional[VoiceSettings]: - """ - Create new voice settings. - - Args: - settingsData: Dictionary containing voice settings data - - Returns: - Created VoiceSettings object or None if failed - """ - try: - logger.info(f"Creating voice settings: {settingsData}") - - # Ensure mandateId is set from context if not provided - if "mandateId" not in settingsData or not settingsData["mandateId"]: - if not self.mandateId: - raise ValueError("mandateId is required but not provided and context has no mandateId") - settingsData["mandateId"] = self.mandateId - - # Add timestamps - currentTime = getUtcTimestamp() - settingsData["creationDate"] = currentTime - settingsData["lastModified"] = currentTime - - # Create VoiceSettings object - voiceSettings = VoiceSettings(**settingsData) - - logger.info(f"✅ Voice settings created: {voiceSettings.id}") - return voiceSettings - - except Exception as e: - logger.error(f"❌ Error creating voice settings: {e}") - return None - - def updateVoiceSettings(self, userId: str, settingsData: Dict[str, Any]) -> Optional[VoiceSettings]: - """ - Update existing voice settings. - - Args: - userId: User ID to update settings for - settingsData: Dictionary containing updated voice settings data - - Returns: - Updated VoiceSettings object or None if failed - """ - try: - logger.info(f"Updating voice settings for user {userId}: {settingsData}") - - # Add last modified timestamp - settingsData["lastModified"] = getUtcTimestamp() - - # Create updated VoiceSettings object - voiceSettings = VoiceSettings(**settingsData) - - logger.info(f"✅ Voice settings updated: {voiceSettings.id}") - return voiceSettings - - except Exception as e: - logger.error(f"❌ Error updating voice settings: {e}") - return None - - def getOrCreateVoiceSettings(self, userId: str) -> Optional[VoiceSettings]: - """ - Get existing voice settings or create default ones. - - Args: - userId: User ID to get/create settings for - - Returns: - VoiceSettings object - """ - try: - # Try to get existing settings - existingSettings = self.getVoiceSettings(userId) - - if existingSettings: - return existingSettings - - # Create default settings if none exist - defaultSettings = { - "userId": userId, - "mandateId": self.mandateId, - "sttLanguage": "de-DE", - "ttsLanguage": "de-DE", - "ttsVoice": "de-DE-Wavenet-A", - "translationEnabled": True, - "targetLanguage": "en-US" - } - - return self.createVoiceSettings(defaultSettings) - - except Exception as e: - logger.error(f"❌ Error getting or creating voice settings: {e}") - return None - # Language and Voice Information async def getAvailableLanguages(self) -> Dict[str, Any]: diff --git a/modules/migration/__init__.py b/modules/migration/__init__.py new file mode 100644 index 00000000..7639be60 --- /dev/null +++ b/modules/migration/__init__.py @@ -0,0 +1 @@ +# Migration modules diff --git a/modules/migration/migrateRagScopeFields.py b/modules/migration/migrateRagScopeFields.py new file mode 100644 index 00000000..82e0e3fb --- /dev/null +++ b/modules/migration/migrateRagScopeFields.py @@ -0,0 +1,114 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Migration: Backfill FileContentIndex scope fields from FileItem (Single Source of Truth). + +Fixes legacy rows in poweron_knowledge where scope/mandateId/featureInstanceId +are empty or default ("personal") despite the corresponding FileItem having correct values. + +Idempotent — safe to run multiple times. Uses a DB flag to skip if already completed. +""" + +import logging +from modules.shared.configuration import APP_CONFIG +from modules.connectors.connectorDbPostgre import _get_cached_connector + +logger = logging.getLogger(__name__) + +_MIGRATION_FLAG_KEY = "migration_rag_scope_fields_completed" + + +def _isMigrationCompleted(appDb) -> bool: + try: + from modules.datamodels.datamodelUam import Mandate + records = appDb.getRecordset(Mandate, recordFilter={"name": _MIGRATION_FLAG_KEY}) + return len(records) > 0 + except Exception: + return False + + +def _setMigrationCompleted(appDb) -> None: + try: + from modules.datamodels.datamodelUam import Mandate + flag = Mandate(name=_MIGRATION_FLAG_KEY, description="RAG scope fields migration completed") + appDb.recordCreate(Mandate, flag) + except Exception as e: + logger.error("Could not set migration flag: %s", e) + + +def runMigration(appDb=None) -> dict: + """Backfill FileContentIndex rows from FileItem metadata. + + Returns dict with counts: {total, updated, skipped, orphaned}. + """ + from modules.datamodels.datamodelKnowledge import FileContentIndex + from modules.datamodels.datamodelFiles import FileItem + from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface + from modules.interfaces.interfaceDbManagement import ComponentObjects + + if appDb is None: + from modules.interfaces.interfaceDbApp import getRootInterface + appDb = getRootInterface().db + + if _isMigrationCompleted(appDb): + logger.info("migrateRagScopeFields: already completed, skipping") + return {"total": 0, "updated": 0, "skipped": 0, "orphaned": 0} + + knowDb = getKnowledgeInterface(None).db + mgmtDb = ComponentObjects().db + + allIndexes = knowDb.getRecordset(FileContentIndex, recordFilter={}) + total = len(allIndexes) + updated = 0 + skipped = 0 + orphaned = 0 + + logger.info("migrateRagScopeFields: processing %d FileContentIndex rows", total) + + for idx in allIndexes: + idxId = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if not idxId: + skipped += 1 + continue + + fileItem = mgmtDb._loadRecord(FileItem, str(idxId)) + if not fileItem: + orphaned += 1 + continue + + _get = (lambda k, d="": fileItem.get(k, d)) if isinstance(fileItem, dict) else (lambda k, d="": getattr(fileItem, k, d)) + + fiScope = _get("scope") or "personal" + fiMandateId = str(_get("mandateId") or "") + fiFeatureInstanceId = str(_get("featureInstanceId") or "") + + idxGet = (lambda k, d="": idx.get(k, d)) if isinstance(idx, dict) else (lambda k, d="": getattr(idx, k, d)) + currentScope = idxGet("scope") or "personal" + currentMandateId = str(idxGet("mandateId") or "") + currentFeatureInstanceId = str(idxGet("featureInstanceId") or "") + + updates = {} + if fiScope != currentScope: + updates["scope"] = fiScope + if fiMandateId and fiMandateId != currentMandateId: + updates["mandateId"] = fiMandateId + if fiFeatureInstanceId and fiFeatureInstanceId != currentFeatureInstanceId: + updates["featureInstanceId"] = fiFeatureInstanceId + + if updates: + try: + knowDb.recordModify(FileContentIndex, str(idxId), updates) + updated += 1 + logger.debug("migrateRagScopeFields: updated %s -> %s", idxId, updates) + except Exception as e: + logger.error("migrateRagScopeFields: failed to update %s: %s", idxId, e) + skipped += 1 + else: + skipped += 1 + + _setMigrationCompleted(appDb) + logger.info( + "migrateRagScopeFields complete: total=%d, updated=%d, skipped=%d, orphaned=%d", + total, updated, skipped, orphaned, + ) + return {"total": total, "updated": updated, "skipped": skipped, "orphaned": orphaned} diff --git a/modules/migration/migrateRootUsers.py b/modules/migration/migrateRootUsers.py new file mode 100644 index 00000000..11424987 --- /dev/null +++ b/modules/migration/migrateRootUsers.py @@ -0,0 +1,329 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Migration: Root-Mandant bereinigen. +Moves all end-user data from Root mandate shared instances to own mandates. +Called once from bootstrap, sets a DB flag to prevent re-execution. +""" + +import logging +from typing import Optional, List, Dict, Any + +logger = logging.getLogger(__name__) + +_MIGRATION_FLAG_KEY = "migration_root_users_completed" + +_DATA_TABLES = [ + "ChatWorkflow", + "FileItem", + "DataSource", + "DataNeutralizerAttributes", + "FileContentIndex", +] + + +def _isMigrationCompleted(db) -> bool: + """Check if migration has already been executed.""" + try: + from modules.datamodels.datamodelUam import Mandate + records = db.getRecordset(Mandate, recordFilter={"name": _MIGRATION_FLAG_KEY}) + return len(records) > 0 + except Exception: + return False + + +def _setMigrationCompleted(db) -> None: + """Set flag that migration is completed (uses a settings-like record).""" + if _isMigrationCompleted(db): + return + try: + from modules.datamodels.datamodelUam import Mandate + flag = Mandate(name=_MIGRATION_FLAG_KEY, label="Migration completed", enabled=False, isSystem=True) + db.recordCreate(Mandate, flag) + logger.info("Migration flag set: root user migration completed") + except Exception as e: + logger.error(f"Failed to set migration flag: {e}") + + +def _findOrCreateTargetInstance(db, featureInterface, featureCode: str, targetMandateId: str, rootInstance: dict) -> dict: + """Find existing or create new FeatureInstance in target mandate. Idempotent.""" + from modules.datamodels.datamodelFeatures import FeatureInstance + + existing = db.getRecordset(FeatureInstance, recordFilter={ + "featureCode": featureCode, + "mandateId": targetMandateId, + }) + if existing: + logger.debug(f"Target instance already exists for {featureCode} in mandate {targetMandateId}") + return existing[0] + + label = rootInstance.get("label") or featureCode + instance = featureInterface.createFeatureInstance( + featureCode=featureCode, + mandateId=targetMandateId, + label=label, + enabled=True, + copyTemplateRoles=True, + ) + if isinstance(instance, dict): + return instance + return instance.model_dump() if hasattr(instance, "model_dump") else {"id": instance.id} + + +def _migrateDataRecords(db, oldInstanceId: str, newInstanceId: str, userId: str) -> int: + """Bulk-update featureInstanceId on all data tables for records owned by userId.""" + totalMigrated = 0 + db._ensure_connection() + for tableName in _DATA_TABLES: + try: + with db.connection.cursor() as cursor: + cursor.execute( + f'UPDATE "{tableName}" ' + f'SET "featureInstanceId" = %s ' + f'WHERE "featureInstanceId" = %s AND "sysCreatedBy" = %s', + (newInstanceId, oldInstanceId, userId), + ) + count = cursor.rowcount + db.connection.commit() + if count > 0: + logger.info(f" Migrated {count} rows in {tableName}: {oldInstanceId} -> {newInstanceId}") + totalMigrated += count + except Exception as e: + try: + db.connection.rollback() + except Exception: + pass + logger.debug(f" Table {tableName} skipped (may not exist or no matching column): {e}") + return totalMigrated + + +def _grantFeatureAccess(db, userId: str, featureInstanceId: str) -> dict: + """Create FeatureAccess + admin role on a feature instance. Idempotent.""" + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + from modules.datamodels.datamodelRbac import Role + + existing = db.getRecordset(FeatureAccess, recordFilter={ + "userId": userId, + "featureInstanceId": featureInstanceId, + }) + if existing: + logger.debug(f"FeatureAccess already exists for user {userId} on instance {featureInstanceId}") + return existing[0] + + fa = FeatureAccess(userId=userId, featureInstanceId=featureInstanceId, enabled=True) + createdFa = db.recordCreate(FeatureAccess, fa.model_dump()) + if not createdFa: + logger.warning(f"Failed to create FeatureAccess for user {userId} on instance {featureInstanceId}") + return {} + + instanceRoles = db.getRecordset(Role, recordFilter={"featureInstanceId": featureInstanceId}) + adminRoleId = None + for r in instanceRoles: + roleLabel = (r.get("roleLabel") or "").lower() + if roleLabel.endswith("-admin"): + adminRoleId = r.get("id") + break + if not adminRoleId: + raise ValueError( + f"No feature-specific admin role for instance {featureInstanceId}. " + f"Cannot create FeatureAccess without role — even in migration context." + ) + far = FeatureAccessRole(featureAccessId=createdFa["id"], roleId=adminRoleId) + db.recordCreate(FeatureAccessRole, far.model_dump()) + + return createdFa + + +def migrateRootUsers(db, dryRun: bool = False) -> dict: + """ + Migrate all end-user feature data from Root mandate to personal mandates. + + Algorithm: + STEP 1: For each user with FeatureAccess on Root instances: + - If user has own mandate: target = existing mandate + - If not: create personal mandate via _provisionMandateForUser + - For each FeatureAccess: create new instance in target, migrate data, transfer access + + STEP 2: Clean up Root: + - Delete all FeatureInstances in Root + - Remove UserMandate for non-sysadmin users + + Args: + db: Database connector + dryRun: If True, log actions without making changes + + Returns: + Summary dict with migration statistics + """ + if _isMigrationCompleted(db): + logger.info("Root user migration already completed, skipping") + return {"status": "already_completed"} + + from modules.datamodels.datamodelUam import Mandate, User, UserInDB + from modules.datamodels.datamodelMembership import ( + UserMandate, UserMandateRole, FeatureAccess, FeatureAccessRole, + ) + from modules.datamodels.datamodelFeatures import FeatureInstance + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.interfaces.interfaceFeatures import getFeatureInterface + + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(db) + stats = { + "usersProcessed": 0, + "mandatesCreated": 0, + "instancesMigrated": 0, + "dataRowsMigrated": 0, + "rootInstancesDeleted": 0, + "rootMembershipsRemoved": 0, + "dryRun": dryRun, + } + + # Find root mandate + rootMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True}) + if not rootMandates: + logger.warning("No root mandate found, nothing to migrate") + return {"status": "no_root_mandate"} + rootMandateId = rootMandates[0].get("id") + + # Get all feature instances in root + rootInstances = db.getRecordset(FeatureInstance, recordFilter={"mandateId": rootMandateId}) + if not rootInstances: + logger.info("No feature instances in root mandate, nothing to migrate") + if not dryRun: + _setMigrationCompleted(db) + return {"status": "no_instances", **stats} + + # Get all FeatureAccess on root instances + rootInstanceIds = {inst.get("id") for inst in rootInstances} + + # Collect unique users with access on root instances + usersToMigrate = {} + for instanceId in rootInstanceIds: + accesses = db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instanceId}) + for access in accesses: + userId = access.get("userId") + if userId not in usersToMigrate: + usersToMigrate[userId] = [] + usersToMigrate[userId].append({ + "featureAccessId": access.get("id"), + "featureInstanceId": instanceId, + }) + + logger.info(f"Migration: {len(usersToMigrate)} users with {sum(len(v) for v in usersToMigrate.values())} accesses on {len(rootInstances)} root instances") + + # STEP 1: Migrate users + for userId, accessList in usersToMigrate.items(): + try: + # Find user + users = db.getRecordset(UserInDB, recordFilter={"id": userId}) + if not users: + logger.warning(f"User {userId} not found, skipping") + continue + user = users[0] + username = user.get("username", "unknown") + + # Check if user has own non-root mandate + userMandates = db.getRecordset(UserMandate, recordFilter={"userId": userId, "enabled": True}) + targetMandateId = None + for um in userMandates: + mid = um.get("mandateId") + if mid != rootMandateId: + targetMandateId = mid + break + + if not targetMandateId: + # Create personal mandate + if dryRun: + logger.info(f"[DRY RUN] Would create personal mandate for user {username}") + stats["mandatesCreated"] += 1 + else: + try: + result = rootInterface._provisionMandateForUser( + userId=userId, + mandateName=f"Home {username}", + planKey="TRIAL_7D", + ) + targetMandateId = result["mandateId"] + stats["mandatesCreated"] += 1 + logger.info(f"Created personal mandate {targetMandateId} for user {username}") + except Exception as e: + logger.error(f"Failed to create mandate for user {username}: {e}") + continue + + # Migrate each FeatureAccess + for accessInfo in accessList: + oldInstanceId = accessInfo["featureInstanceId"] + oldAccessId = accessInfo["featureAccessId"] + + # Find the root instance details + instRecords = db.getRecordset(FeatureInstance, recordFilter={"id": oldInstanceId}) + if not instRecords: + continue + featureCode = instRecords[0].get("featureCode") + + if dryRun: + logger.info(f"[DRY RUN] Would migrate {featureCode} for {username} to mandate {targetMandateId}") + stats["instancesMigrated"] += 1 + else: + targetInstance = _findOrCreateTargetInstance( + db, featureInterface, featureCode, targetMandateId, instRecords[0], + ) + newInstanceId = targetInstance.get("id") + if not newInstanceId: + logger.error(f"Failed to obtain target instance for {featureCode} in mandate {targetMandateId}") + continue + + migratedCount = _migrateDataRecords(db, oldInstanceId, newInstanceId, userId) + + _grantFeatureAccess(db, userId, newInstanceId) + + try: + db.recordDelete(FeatureAccess, oldAccessId) + except Exception as delErr: + logger.warning(f"Could not remove old FeatureAccess {oldAccessId}: {delErr}") + + logger.info( + f"Migrated {featureCode} for {username}: " + f"instance {oldInstanceId} -> {newInstanceId}, {migratedCount} data rows moved" + ) + stats["instancesMigrated"] += 1 + stats["dataRowsMigrated"] += migratedCount + + stats["usersProcessed"] += 1 + + except Exception as e: + logger.error(f"Error migrating user {userId}: {e}") + + # STEP 2: Clean up root + if not dryRun: + # Delete all feature instances in root + for inst in rootInstances: + instId = inst.get("id") + try: + # First delete all FeatureAccess on this instance + accesses = db.getRecordset(FeatureAccess, recordFilter={"featureInstanceId": instId}) + for access in accesses: + db.recordDelete(FeatureAccess, access.get("id")) + db.recordDelete(FeatureInstance, instId) + stats["rootInstancesDeleted"] += 1 + except Exception as e: + logger.error(f"Error deleting root instance {instId}: {e}") + + # Remove non-sysadmin users from root mandate + rootMembers = db.getRecordset(UserMandate, recordFilter={"mandateId": rootMandateId}) + for membership in rootMembers: + membUserId = membership.get("userId") + userRecords = db.getRecordset(UserInDB, recordFilter={"id": membUserId}) + if userRecords and userRecords[0].get("isSysAdmin"): + continue + try: + db.recordDelete(UserMandate, membership.get("id")) + stats["rootMembershipsRemoved"] += 1 + except Exception as e: + logger.error(f"Error removing root membership for {membUserId}: {e}") + + _setMigrationCompleted(db) + + logger.info(f"Migration completed: {stats}") + return {"status": "completed", **stats} diff --git a/modules/migration/migrateVoiceAndDocuments.py b/modules/migration/migrateVoiceAndDocuments.py new file mode 100644 index 00000000..0fc5ee02 --- /dev/null +++ b/modules/migration/migrateVoiceAndDocuments.py @@ -0,0 +1,316 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Migration: Voice settings consolidation and CoachingDocument scope-tagging. +Moves VoiceSettings (workspace DB) and CoachingUserProfile voice fields (commcoach DB) +into the unified UserVoicePreferences model, and tags CoachingDocument files with +featureInstance scope before deleting the legacy records. +Called once from bootstrap, sets a DB flag to prevent re-execution. +""" + +import logging +import uuid +from typing import Dict, List, Optional + +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +from modules.datamodels.datamodelUam import UserVoicePreferences + +logger = logging.getLogger(__name__) + +_MIGRATION_FLAG_KEY = "migration_voice_documents_completed" + + +def _isMigrationCompleted(db) -> bool: + """Check if migration has already been executed.""" + try: + from modules.datamodels.datamodelUam import Mandate + records = db.getRecordset(Mandate, recordFilter={"name": _MIGRATION_FLAG_KEY}) + return len(records) > 0 + except Exception: + return False + + +def _setMigrationCompleted(db) -> None: + """Set flag that migration is completed (uses a settings-like record).""" + if _isMigrationCompleted(db): + return + try: + from modules.datamodels.datamodelUam import Mandate + flag = Mandate(name=_MIGRATION_FLAG_KEY, label="Migration completed", enabled=False, isSystem=True) + db.recordCreate(Mandate, flag) + logger.info("Migration flag set: voice & documents migration completed") + except Exception as e: + logger.error(f"Failed to set migration flag: {e}") + + +def _getRawRows(connector: DatabaseConnector, tableName: str, columns: List[str]) -> List[Dict]: + """Read all rows from a table via raw SQL. Returns empty list if table doesn't exist.""" + try: + connector._ensure_connection() + colList = ", ".join(f'"{c}"' for c in columns) + with connector.connection.cursor() as cur: + cur.execute( + "SELECT COUNT(*) FROM information_schema.tables " + "WHERE LOWER(table_name) = LOWER(%s) AND table_schema = 'public'", + (tableName,), + ) + if cur.fetchone()["count"] == 0: + logger.info(f"Table '{tableName}' does not exist, skipping") + return [] + cur.execute(f'SELECT {colList} FROM "{tableName}"') + return [dict(row) for row in cur.fetchall()] + except Exception as e: + logger.warning(f"Raw query on '{tableName}' failed: {e}") + try: + connector.connection.rollback() + except Exception: + pass + return [] + + +def _deleteRawRow(connector: DatabaseConnector, tableName: str, rowId: str) -> bool: + """Delete a single row by id via raw SQL.""" + try: + connector._ensure_connection() + with connector.connection.cursor() as cur: + cur.execute(f'DELETE FROM "{tableName}" WHERE "id" = %s', (rowId,)) + connector.connection.commit() + return True + except Exception as e: + logger.warning(f"Failed to delete row {rowId} from '{tableName}': {e}") + try: + connector.connection.rollback() + except Exception: + pass + return False + + +def _createDbConnector(dbName: str) -> Optional[DatabaseConnector]: + """Create a DatabaseConnector for a named database, returns None on failure.""" + try: + dbHost = APP_CONFIG.get("DB_HOST") + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + return DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbName, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + ) + except Exception as e: + logger.warning(f"Could not connect to database '{dbName}': {e}") + return None + + +# ─── Part A ─────────────────────────────────────────────────────────────────── + +def _migrateVoiceSettings(db, wsDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Migrate VoiceSettings records from poweron_workspace into UserVoicePreferences.""" + rows = _getRawRows(wsDb, "VoiceSettings", [ + "id", "userId", "mandateId", "ttsVoiceMap", "sttLanguage", "ttsLanguage", "ttsVoice", + ]) + if not rows: + logger.info("Part A: No VoiceSettings records found, skipping") + return + + for row in rows: + userId = row.get("userId") + if not userId: + continue + + existing = db.getRecordset(UserVoicePreferences, recordFilter={"userId": userId}) + if existing: + stats["voiceSettingsSkipped"] += 1 + if not dryRun: + _deleteRawRow(wsDb, "VoiceSettings", row["id"]) + continue + + if dryRun: + logger.info(f"[DRY RUN] Would create UserVoicePreferences for user {userId} from VoiceSettings") + stats["voiceSettingsCreated"] += 1 + continue + + try: + import json + ttsVoiceMap = row.get("ttsVoiceMap") + if isinstance(ttsVoiceMap, str): + try: + ttsVoiceMap = json.loads(ttsVoiceMap) + except (json.JSONDecodeError, TypeError): + ttsVoiceMap = None + + prefs = UserVoicePreferences( + userId=userId, + mandateId=row.get("mandateId"), + ttsVoiceMap=ttsVoiceMap, + sttLanguage=row.get("sttLanguage", "de-DE"), + ttsLanguage=row.get("ttsLanguage", "de-DE"), + ttsVoice=row.get("ttsVoice"), + ) + db.recordCreate(UserVoicePreferences, prefs) + stats["voiceSettingsCreated"] += 1 + _deleteRawRow(wsDb, "VoiceSettings", row["id"]) + except Exception as e: + logger.error(f"Part A: Failed to migrate VoiceSettings {row['id']}: {e}") + stats["errors"] += 1 + + +# ─── Part B ─────────────────────────────────────────────────────────────────── + +def _migrateCoachingProfileVoice(db, ccDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Migrate preferredLanguage/preferredVoice from CoachingUserProfile into UserVoicePreferences.""" + rows = _getRawRows(ccDb, "CoachingUserProfile", [ + "id", "userId", "mandateId", "preferredLanguage", "preferredVoice", + ]) + if not rows: + logger.info("Part B: No CoachingUserProfile records with voice data found, skipping") + return + + for row in rows: + userId = row.get("userId") + prefLang = row.get("preferredLanguage") + prefVoice = row.get("preferredVoice") + if not userId or (not prefLang and not prefVoice): + continue + + existing = db.getRecordset(UserVoicePreferences, recordFilter={"userId": userId}) + if existing: + stats["coachingProfileSkipped"] += 1 + continue + + if dryRun: + logger.info(f"[DRY RUN] Would create UserVoicePreferences for user {userId} from CoachingUserProfile") + stats["coachingProfileCreated"] += 1 + continue + + try: + prefs = UserVoicePreferences( + userId=userId, + mandateId=row.get("mandateId"), + sttLanguage=prefLang or "de-DE", + ttsLanguage=prefLang or "de-DE", + ttsVoice=prefVoice, + ) + db.recordCreate(UserVoicePreferences, prefs) + stats["coachingProfileCreated"] += 1 + except Exception as e: + logger.error(f"Part B: Failed to migrate CoachingUserProfile {row['id']}: {e}") + stats["errors"] += 1 + + +# ─── Part C ─────────────────────────────────────────────────────────────────── + +def _migrateCoachingDocuments(ccDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Tag FileItem/FileContentIndex with featureInstance scope for each CoachingDocument.""" + from modules.datamodels.datamodelFiles import FileItem + from modules.datamodels.datamodelKnowledge import FileContentIndex + + rows = _getRawRows(ccDb, "CoachingDocument", [ + "id", "fileRef", "instanceId", + ]) + if not rows: + logger.info("Part C: No CoachingDocument records found, skipping") + return + + mgmtDb = _createDbConnector("poweron_management") + knowledgeDb = _createDbConnector("poweron_knowledge") + if not mgmtDb: + logger.error("Part C: Cannot connect to poweron_management, aborting document migration") + return + + for row in rows: + fileRef = row.get("fileRef") + instanceId = row.get("instanceId") + docId = row.get("id") + if not fileRef: + if not dryRun: + _deleteRawRow(ccDb, "CoachingDocument", docId) + continue + + if dryRun: + logger.info(f"[DRY RUN] Would tag FileItem {fileRef} with featureInstanceId={instanceId}") + stats["documentsTagged"] += 1 + continue + + try: + fileRecords = mgmtDb.getRecordset(FileItem, recordFilter={"id": fileRef}) + if fileRecords: + updateData = {"scope": "featureInstance"} + if instanceId: + updateData["featureInstanceId"] = instanceId + mgmtDb.recordModify(FileItem, fileRef, updateData) + stats["documentsTagged"] += 1 + else: + logger.warning(f"Part C: FileItem {fileRef} not found in management DB") + + if knowledgeDb: + fciRecords = knowledgeDb.getRecordset(FileContentIndex, recordFilter={"id": fileRef}) + if fciRecords: + fciUpdate = {"scope": "featureInstance"} + if instanceId: + fciUpdate["featureInstanceId"] = instanceId + knowledgeDb.recordModify(FileContentIndex, fileRef, fciUpdate) + + _deleteRawRow(ccDb, "CoachingDocument", docId) + except Exception as e: + logger.error(f"Part C: Failed to migrate CoachingDocument {docId}: {e}") + stats["errors"] += 1 + + +# ─── Main entry ─────────────────────────────────────────────────────────────── + +def migrateVoiceAndDocuments(db, dryRun: bool = False) -> dict: + """ + Migrate VoiceSettings + CoachingUserProfile voice fields into UserVoicePreferences, + and tag CoachingDocument files with featureInstance scope. + + Args: + db: Root database connector (poweron_app) + dryRun: If True, log actions without making changes + + Returns: + Summary dict with migration statistics + """ + if _isMigrationCompleted(db): + logger.info("Voice & documents migration already completed, skipping") + return {"status": "already_completed"} + + stats = { + "voiceSettingsCreated": 0, + "voiceSettingsSkipped": 0, + "coachingProfileCreated": 0, + "coachingProfileSkipped": 0, + "documentsTagged": 0, + "errors": 0, + "dryRun": dryRun, + } + + wsDb = _createDbConnector("poweron_workspace") + ccDb = _createDbConnector("poweron_commcoach") + + # Part A + if wsDb: + _migrateVoiceSettings(db, wsDb, dryRun, stats) + else: + logger.warning("Skipping Part A: poweron_workspace DB unavailable") + + # Part B + if ccDb: + _migrateCoachingProfileVoice(db, ccDb, dryRun, stats) + else: + logger.warning("Skipping Part B: poweron_commcoach DB unavailable") + + # Part C + if ccDb: + _migrateCoachingDocuments(ccDb, dryRun, stats) + else: + logger.warning("Skipping Part C: poweron_commcoach DB unavailable") + + if not dryRun: + _setMigrationCompleted(db) + + logger.info(f"Voice & documents migration completed: {stats}") + return {"status": "completed", **stats} diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index 47d3ac9c..553c66d3 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -112,12 +112,12 @@ def _buildEnrichedAutomationEvents(currentUser: User) -> List[Dict[str, Any]]: if automation: if isinstance(automation, dict): job["name"] = automation.get("label", "") - job["createdBy"] = _resolveUsername(automation.get("_createdBy", "")) + job["createdBy"] = _resolveUsername(automation.get("sysCreatedBy", "")) job["mandate"] = _resolveMandateLabel(automation.get("mandateId", "")) job["featureInstance"] = _resolveFeatureLabel(automation.get("featureInstanceId", "")) else: job["name"] = getattr(automation, "label", "") - job["createdBy"] = _resolveUsername(getattr(automation, "_createdBy", "")) + job["createdBy"] = _resolveUsername(getattr(automation, "sysCreatedBy", "")) job["mandate"] = _resolveMandateLabel(getattr(automation, "mandateId", "")) job["featureInstance"] = _resolveFeatureLabel(getattr(automation, "featureInstanceId", "")) else: diff --git a/modules/routes/routeAdminAutomationLogs.py b/modules/routes/routeAdminAutomationLogs.py index 8b4d897b..479d0df3 100644 --- a/modules/routes/routeAdminAutomationLogs.py +++ b/modules/routes/routeAdminAutomationLogs.py @@ -91,14 +91,14 @@ def _buildFlattenedExecutionLogs(currentUser: User) -> List[Dict[str, Any]]: automationLabel = automation.get("label", "") mandateId = automation.get("mandateId", "") featureInstanceId = automation.get("featureInstanceId", "") - createdBy = automation.get("_createdBy", "") + createdBy = automation.get("sysCreatedBy", "") logs = automation.get("executionLogs") or [] else: automationId = getattr(automation, "id", "") automationLabel = getattr(automation, "label", "") mandateId = getattr(automation, "mandateId", "") featureInstanceId = getattr(automation, "featureInstanceId", "") - createdBy = getattr(automation, "_createdBy", "") + createdBy = getattr(automation, "sysCreatedBy", "") logs = getattr(automation, "executionLogs", None) or [] mandateName = _resolveMandateLabel(mandateId) diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index c95c0b1b..e69df7b9 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -576,14 +576,15 @@ def create_feature_instance( config=data.config ) - # Sync Stripe quantity after successful creation try: from modules.interfaces.interfaceDbSubscription import getInterface as _getSubIf2 from modules.security.rootAccess import getRootUser as _getRU _subIf2 = _getSubIf2(_getRU(), mandateIdStr) - _subIf2.syncQuantityToStripe(mandateIdStr) - except Exception: - pass + _operative = _subIf2.getOperativeForMandate(mandateIdStr) + if _operative: + _subIf2.syncQuantityToStripe(_operative["id"], raiseOnError=True) + except Exception as e: + logger.error("Stripe quantity sync failed for admin feature creation in mandate %s: %s", mandateIdStr, e) logger.info( f"User {context.user.id} created feature instance '{data.label}' " @@ -1172,31 +1173,29 @@ def add_user_to_feature_instance( detail=f"User '{data.userId}' not found" ) - # Check if user already has access - from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole - existingAccess = rootInterface.getFeatureAccess(data.userId, instanceId) - if existingAccess: + if not data.roleIds: raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail="User already has access to this feature instance" + status_code=status.HTTP_400_BAD_REQUEST, + detail="At least one role is required to grant feature access" ) - - # Create FeatureAccess record - featureAccess = FeatureAccess( + + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) + validRoleIds = {r.get("id") for r in instanceRoles} + invalidRoles = [rid for rid in data.roleIds if rid not in validRoleIds] + if invalidRoles: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role IDs {invalidRoles} do not belong to feature instance {instanceId}. " + f"Only instance-scoped roles are allowed, never mandate roles." + ) + + featureAccess = rootInterface.createFeatureAccess( userId=data.userId, featureInstanceId=instanceId, - enabled=True + roleIds=data.roleIds ) - createdAccess = rootInterface.db.recordCreate(FeatureAccess, featureAccess.model_dump()) - featureAccessId = createdAccess.get("id") - - # Create FeatureAccessRole records for each role - for roleId in data.roleIds: - featureAccessRole = FeatureAccessRole( - featureAccessId=featureAccessId, - roleId=roleId - ) - rootInterface.db.recordCreate(FeatureAccessRole, featureAccessRole.model_dump()) + featureAccessId = str(featureAccess.id) logger.info( f"User {context.user.id} added user {data.userId} to feature instance {instanceId} " @@ -1379,10 +1378,19 @@ def update_feature_instance_user_roles( if data.enabled is not None: rootInterface.db.recordModify(FeatureAccess, featureAccessId, {"enabled": data.enabled}) - # Delete existing FeatureAccessRole records via interface method + from modules.datamodels.datamodelRbac import Role + instanceRoles = rootInterface.db.getRecordset(Role, recordFilter={"featureInstanceId": instanceId}) + validRoleIds = {r.get("id") for r in instanceRoles} + invalidRoles = [rid for rid in data.roleIds if rid not in validRoleIds] + if invalidRoles: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Role IDs {invalidRoles} do not belong to feature instance {instanceId}. " + f"Only instance-scoped roles are allowed, never mandate roles." + ) + rootInterface.deleteFeatureAccessRoles(featureAccessId) - # Create new FeatureAccessRole records for roleId in data.roleIds: featureAccessRole = FeatureAccessRole( featureAccessId=featureAccessId, @@ -1523,6 +1531,65 @@ def get_feature( ) +# ============================================================================= +# Instance Rename (for instance admins, used by navigation tree) +# ============================================================================= + +class FeatureInstanceRenameRequest(BaseModel): + """Request model for renaming a feature instance""" + label: str = Field(..., min_length=1, max_length=200, description="New label for the instance") + + +@router.patch("/instances/{instanceId}/rename", response_model=Dict[str, Any]) +@limiter.limit("30/minute") +def _renameFeatureInstance( + request: Request, + instanceId: str, + data: FeatureInstanceRenameRequest, + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """ + Rename a feature instance. Requires instance admin role. + """ + try: + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Feature instance not found") + + userId = str(context.user.id) + isInstanceAdmin = False + if context.hasSysAdminRole: + isInstanceAdmin = True + else: + from modules.datamodels.datamodelMembership import FeatureAccess, FeatureAccessRole + fa = rootInterface.getFeatureAccess(userId, instanceId) + if fa: + faRoleIds = rootInterface.getRoleIdsForFeatureAccess(str(fa.id)) + for rid in faRoleIds: + role = rootInterface.getRole(rid) + if role and (role.roleLabel or "").lower().endswith("-admin"): + isInstanceAdmin = True + break + + if not isInstanceAdmin: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Instance admin role required to rename") + + updated = featureInterface.updateFeatureInstance(instanceId, {"label": data.label.strip()}) + if not updated: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to update instance") + + return {"id": instanceId, "label": updated.label} + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error renaming feature instance {instanceId}: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + # ============================================================================= # Helper Functions # ============================================================================= diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index 3778d227..16336fae 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -1477,7 +1477,7 @@ def cleanup_duplicate_access_rules( for sig, rules in rulesBySignature.items(): if len(rules) > 1: # Sort by creation time (keep oldest) - rules.sort(key=lambda r: r.get("_createdAt", 0)) + rules.sort(key=lambda r: r.get("sysCreatedAt", 0)) keepRule = rules[0] deleteRules = rules[1:] diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index 04412752..0f612d45 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Resp from typing import List, Dict, Any, Optional from fastapi import status import logging -from datetime import date, datetime +from datetime import date, datetime, timezone from pydantic import BaseModel, Field # Import auth module @@ -30,7 +30,6 @@ from modules.datamodels.datamodelBilling import ( BillingAccount, BillingTransaction, BillingSettings, - BillingModelEnum, TransactionTypeEnum, ReferenceTypeEnum, PeriodTypeEnum, @@ -38,7 +37,6 @@ from modules.datamodels.datamodelBilling import ( BillingStatisticsResponse, BillingStatisticsChartData, BillingCheckResult, - parseBillingModelFromStoredValue, ) # Configure logger @@ -229,14 +227,14 @@ def _filterTransactionsByScope(transactions: list, scope: BillingDataScope) -> l class CreditAddRequest(BaseModel): """Request model for adding or deducting credit from an account.""" - userId: Optional[str] = Field(None, description="Target user ID (for PREPAY_USER model)") + userId: Optional[str] = Field(None, description="Target user ID for audit trail only (optional)") amount: float = Field(..., description="Amount in CHF. Positive = credit, negative = deduction. Must not be zero.") description: str = Field(default="Manual credit", description="Transaction description") class CheckoutCreateRequest(BaseModel): """Request model for creating Stripe Checkout Session.""" - userId: Optional[str] = Field(None, description="Target user ID (for PREPAY_USER model)") + userId: Optional[str] = Field(None, description="Target user ID for audit trail only (optional)") amount: float = Field(..., gt=0, description="Amount to pay in CHF (must be in allowed presets)") returnUrl: str = Field(..., min_length=1, description="Absolute frontend URL used for Stripe success/cancel redirects") @@ -262,11 +260,12 @@ class CheckoutConfirmResponse(BaseModel): class BillingSettingsUpdate(BaseModel): """Request model for updating billing settings.""" - billingModel: Optional[BillingModelEnum] = None - defaultUserCredit: Optional[float] = Field(None, ge=0) warningThresholdPercent: Optional[float] = Field(None, ge=0, le=100) notifyOnWarning: Optional[bool] = None notifyEmails: Optional[List[str]] = None + autoRechargeEnabled: Optional[bool] = None + rechargeAmountCHF: Optional[float] = Field(None, gt=0) + rechargeMaxPerMonth: Optional[int] = Field(None, ge=0) class TransactionResponse(BaseModel): @@ -293,7 +292,6 @@ class AccountSummary(BaseModel): id: str mandateId: str userId: Optional[str] - accountType: str balance: float warningThreshold: float enabled: bool @@ -317,10 +315,8 @@ class MandateBalanceResponse(BaseModel): """Mandate-level balance summary.""" mandateId: str mandateName: str - billingModel: str totalBalance: float userCount: int - defaultUserCredit: float warningThresholdPercent: float @@ -414,15 +410,7 @@ def _creditStripeSessionIfNeeded( if not settings: raise HTTPException(status_code=404, detail="Billing settings not found") - billing_model = parseBillingModelFromStoredValue(settings.get("billingModel")) - if billing_model == BillingModelEnum.PREPAY_USER: - if not user_id: - raise HTTPException(status_code=400, detail="userId required for PREPAY_USER") - account = billingInterface.getOrCreateUserAccount(mandate_id, user_id, initialBalance=0.0) - elif billing_model == BillingModelEnum.PREPAY_MANDATE: - account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0) - else: - raise HTTPException(status_code=400, detail=f"Cannot add credit to {billing_model.value}") + account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0) transaction = BillingTransaction( accountId=account["id"], @@ -516,7 +504,6 @@ def getBalanceForMandate( return BillingBalanceResponse( mandateId=targetMandateId, mandateName=mandateName, - billingModel=checkResult.billingModel or BillingModelEnum.PREPAY_MANDATE, balance=checkResult.currentBalance or 0.0, warningThreshold=0.0, # TODO: Get from account isWarning=False, @@ -564,7 +551,7 @@ def getTransactions( aicoreProvider=t.get("aicoreProvider"), aicoreModel=t.get("aicoreModel"), createdByUserId=t.get("createdByUserId"), - createdAt=t.get("_createdAt"), + createdAt=t.get("sysCreatedAt"), mandateId=t.get("mandateId"), mandateName=t.get("mandateName") )) @@ -608,8 +595,6 @@ def getStatistics( costByFeature={} ) - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - # Transactions are always on user accounts (audit trail) account = billingInterface.getUserAccount(ctx.mandateId, ctx.user.id) @@ -722,11 +707,13 @@ def createOrUpdateSettings( targetMandateId: str = Path(..., description="Mandate ID"), settingsUpdate: BillingSettingsUpdate = Body(...), ctx: RequestContext = Depends(getRequestContext), - _admin = Depends(requireSysAdminRole) ): """ - Create or update billing settings for a mandate (SysAdmin only). + Create or update billing settings for a mandate. + Access: SysAdmin (any mandate) or MandateAdmin (own mandate). """ + if not _isAdminOfMandate(ctx, targetMandateId): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin role required for this mandate") try: billingInterface = getBillingInterface(ctx.user, targetMandateId) existingSettings = billingInterface.getSettings(targetMandateId) @@ -734,18 +721,6 @@ def createOrUpdateSettings( if existingSettings: updates = settingsUpdate.model_dump(exclude_none=True) if updates: - # Check if billing model is changing - trigger budget migration - if "billingModel" in updates: - oldModel = parseBillingModelFromStoredValue(existingSettings.get("billingModel")) - newModel = ( - BillingModelEnum(updates["billingModel"]) - if isinstance(updates["billingModel"], str) - else updates["billingModel"] - ) - if oldModel != newModel: - migrationResult = billingInterface.switchBillingModel(targetMandateId, oldModel, newModel) - logger.info(f"Billing model migration for {targetMandateId}: {migrationResult}") - result = billingInterface.updateSettings(existingSettings["id"], updates) return result or existingSettings return existingSettings @@ -754,16 +729,6 @@ def createOrUpdateSettings( newSettings = BillingSettings( mandateId=targetMandateId, - billingModel=( - settingsUpdate.billingModel - if settingsUpdate.billingModel is not None - else BillingModelEnum.PREPAY_MANDATE - ), - defaultUserCredit=( - settingsUpdate.defaultUserCredit - if settingsUpdate.defaultUserCredit is not None - else 0.0 - ), warningThresholdPercent=( settingsUpdate.warningThresholdPercent if settingsUpdate.warningThresholdPercent is not None @@ -775,6 +740,21 @@ def createOrUpdateSettings( else True ), notifyEmails=settingsUpdate.notifyEmails or [], + autoRechargeEnabled=( + settingsUpdate.autoRechargeEnabled + if settingsUpdate.autoRechargeEnabled is not None + else False + ), + rechargeAmountCHF=( + settingsUpdate.rechargeAmountCHF + if settingsUpdate.rechargeAmountCHF is not None + else 10.0 + ), + rechargeMaxPerMonth=( + settingsUpdate.rechargeMaxPerMonth + if settingsUpdate.rechargeMaxPerMonth is not None + else 3 + ), ) return billingInterface.createSettings(newSettings) @@ -797,34 +777,15 @@ def addCredit( ): """ Add credit to a billing account (SysAdmin only). - For PREPAY_USER model, specify userId. For PREPAY_MANDATE, leave userId empty. """ try: - # Get settings to determine billing model billingInterface = getBillingInterface(ctx.user, targetMandateId) settings = billingInterface.getSettings(targetMandateId) if not settings: raise HTTPException(status_code=404, detail="Billing settings not found for this mandate") - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - # Validate request based on billing model - if billingModel == BillingModelEnum.PREPAY_USER: - if not creditRequest.userId: - raise HTTPException(status_code=400, detail="userId is required for PREPAY_USER model") - - # Create user-level account if needed and add credit - account = billingInterface.getOrCreateUserAccount( - targetMandateId, - creditRequest.userId, - initialBalance=0.0 - ) - elif billingModel == BillingModelEnum.PREPAY_MANDATE: - # Create mandate-level account if needed and add credit - account = billingInterface.getOrCreateMandateAccount(targetMandateId, initialBalance=0.0) - else: - raise HTTPException(status_code=400, detail=f"Cannot add credit to {billingModel.value} billing model") + account = billingInterface.getOrCreateMandateAccount(targetMandateId, initialBalance=0.0) if creditRequest.amount == 0: raise HTTPException(status_code=400, detail="Amount must not be zero") @@ -867,8 +828,7 @@ def createCheckoutSession( ): """ Create Stripe Checkout Session for credit top-up. Returns redirect URL. - RBAC: PREPAY_USER requires mandate membership (user loads own account), - PREPAY_MANDATE requires mandate admin role. + Requires mandate admin role. """ try: billingInterface = getBillingInterface(ctx.user, targetMandateId) @@ -877,20 +837,8 @@ def createCheckoutSession( if not settings: raise HTTPException(status_code=404, detail="Billing settings not found for this mandate") - billingModel = parseBillingModelFromStoredValue(settings.get("billingModel")) - - if billingModel == BillingModelEnum.PREPAY_USER: - if not checkoutRequest.userId: - raise HTTPException(status_code=400, detail="userId is required for PREPAY_USER model") - if str(checkoutRequest.userId) != str(ctx.user.id): - raise HTTPException(status_code=403, detail="Users can only load credit to their own account") - if not _isMemberOfMandate(ctx, targetMandateId): - raise HTTPException(status_code=403, detail="User is not a member of this mandate") - elif billingModel == BillingModelEnum.PREPAY_MANDATE: - if not _isAdminOfMandate(ctx, targetMandateId): - raise HTTPException(status_code=403, detail="Mandate admin role required to load mandate credit") - else: - raise HTTPException(status_code=400, detail=f"Cannot add credit to {billingModel.value} billing model") + if not _isAdminOfMandate(ctx, targetMandateId): + raise HTTPException(status_code=403, detail="Mandate admin role required to load mandate credit") from modules.serviceCenter.services.serviceBilling.stripeCheckout import create_checkout_session redirect_url = create_checkout_session( @@ -944,19 +892,8 @@ def confirmCheckoutSession( if not settings: raise HTTPException(status_code=404, detail="Billing settings not found") - billing_model = parseBillingModelFromStoredValue(settings.get("billingModel")) - if billing_model == BillingModelEnum.PREPAY_USER: - if not user_id: - raise HTTPException(status_code=400, detail="userId required for PREPAY_USER") - if str(user_id) != str(ctx.user.id): - raise HTTPException(status_code=403, detail="Users can only confirm their own payment sessions") - if not _isMemberOfMandate(ctx, mandate_id): - raise HTTPException(status_code=403, detail="User is not a member of this mandate") - elif billing_model == BillingModelEnum.PREPAY_MANDATE: - if not _isAdminOfMandate(ctx, mandate_id): - raise HTTPException(status_code=403, detail="Mandate admin role required") - else: - raise HTTPException(status_code=400, detail=f"Cannot add credit to {billing_model.value}") + if not _isAdminOfMandate(ctx, mandate_id): + raise HTTPException(status_code=403, detail="Mandate admin role required") root_billing_interface = _getRootInterface() return _creditStripeSessionIfNeeded(root_billing_interface, session_dict, eventId=None) @@ -1167,6 +1104,12 @@ def _handleSubscriptionCheckoutCompleted(session, eventId: str) -> None: updatedSub = subInterface.getById(subscriptionRecordId) _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=updatedSub, platformUrl=platformUrl) + try: + billingIf = _getRootInterface() + billingIf.creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") + except Exception as ex: + logger.error("creditSubscriptionBudget on activation failed: %s", ex) + logger.info( "Checkout completed: sub=%s -> %s, mandate=%s, plan=%s", subscriptionRecordId, toStatus.value, mandateId, planKey, @@ -1186,7 +1129,8 @@ def _handleSubscriptionWebhook(event) -> None: from datetime import datetime, timezone obj = event.data.object - stripeSubId = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription") + rawSub = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription") + stripeSubId = rawSub.get("id") if isinstance(rawSub, dict) else rawSub if not stripeSubId: logger.warning("Subscription webhook %s has no subscription ID", event.type) return @@ -1224,9 +1168,14 @@ def _handleSubscriptionWebhook(event) -> None: if stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.SCHEDULED: subInterface.transitionStatus(subId, SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE) subService.invalidateCache(mandateId) - plan = _getPlan(sub.get("planKey", "")) + planKey = sub.get("planKey", "") + plan = _getPlan(planKey) refreshedSub = subInterface.getById(subId) _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedSub, platformUrl=webhookPlatformUrl) + try: + _getRootInterface().creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") + except Exception as ex: + logger.error("creditSubscriptionBudget SCHEDULED->ACTIVE failed: %s", ex) logger.info("SCHEDULED -> ACTIVE for sub %s (mandate %s)", subId, mandateId) elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.PAST_DUE: @@ -1292,6 +1241,25 @@ def _handleSubscriptionWebhook(event) -> None: logger.error("Failed to notify about trial ending: %s", e) elif event.type == "invoice.paid": + period_ts = obj.get("period_start") + periodLabel = "" + if period_ts: + period_start_at = datetime.fromtimestamp(int(period_ts), tz=timezone.utc) + periodLabel = period_start_at.strftime("%Y-%m-%d") + try: + billing_if = _getRootInterface() + billing_if.resetStorageBillingPeriod(mandateId, period_start_at) + billing_if.reconcileMandateStorageBilling(mandateId) + except Exception as ex: + logger.error("Storage billing on invoice.paid failed: %s", ex) + + planKey = sub.get("planKey", "") + try: + billing_if = _getRootInterface() + billing_if.creditSubscriptionBudget(mandateId, planKey, periodLabel=periodLabel or "Periodenverlängerung") + except Exception as ex: + logger.error("creditSubscriptionBudget on invoice.paid failed: %s", ex) + logger.info("Invoice paid for sub %s (mandate %s)", subId, mandateId) return None @@ -1321,7 +1289,6 @@ def getAccounts( id=acc.get("id"), mandateId=acc.get("mandateId"), userId=acc.get("userId"), - accountType=acc.get("accountType"), balance=acc.get("balance", 0.0), warningThreshold=acc.get("warningThreshold", 0.0), enabled=acc.get("enabled", True) @@ -1404,6 +1371,31 @@ def getUsersForMandate( raise HTTPException(status_code=500, detail=str(e)) +def _attachCreatedByUserNamesToTransactionRows(rows: List[Dict[str, Any]]) -> None: + """Resolve createdByUserId to userName using root app interface (sysadmin transaction views).""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + + appRoot = getRootInterface() + userNames: Dict[str, str] = {} + for row in rows: + uid = row.get("createdByUserId") + if not uid: + row["userName"] = "" + continue + if uid not in userNames: + try: + u = appRoot.getUser(uid) + userNames[uid] = u.username if u else uid[:8] + except Exception: + userNames[uid] = uid[:8] + row["userName"] = userNames.get(uid, "") + except Exception: + for row in rows: + uid = row.get("createdByUserId") + row["userName"] = uid[:8] if uid else "" + + def _enrichTransactionRows(transactions) -> List[Dict[str, Any]]: """Convert raw transaction dicts to enriched TransactionResponse rows with resolved usernames.""" result = [] @@ -1421,27 +1413,11 @@ def _enrichTransactionRows(transactions) -> List[Dict[str, Any]]: aicoreProvider=t.get("aicoreProvider"), aicoreModel=t.get("aicoreModel"), createdByUserId=t.get("createdByUserId"), - createdAt=t.get("_createdAt") + createdAt=t.get("sysCreatedAt") ) result.append(row.model_dump()) - try: - from modules.interfaces.interfaceDbUam import _getRootInterface as getUamRoot - uamInterface = getUamRoot() - userNames: Dict[str, str] = {} - for row in result: - uid = row.get("createdByUserId") - if uid and uid not in userNames: - try: - user = uamInterface.getUser(uid) - userNames[uid] = user.get("username", uid[:8]) if user else uid[:8] - except Exception: - userNames[uid] = uid[:8] - row["userName"] = userNames.get(uid, "") if uid else "" - except Exception: - for row in result: - row["userName"] = row.get("createdByUserId", "")[:8] if row.get("createdByUserId") else "" - + _attachCreatedByUserNamesToTransactionRows(result) return result @@ -1465,28 +1441,11 @@ def _buildTransactionsList(ctx: RequestContext, targetMandateId: str) -> List[Di aicoreProvider=t.get("aicoreProvider"), aicoreModel=t.get("aicoreModel"), createdByUserId=t.get("createdByUserId"), - createdAt=t.get("_createdAt") + createdAt=t.get("sysCreatedAt") ) result.append(row.model_dump()) - # Resolve user names - try: - from modules.interfaces.interfaceDbUam import _getRootInterface as getUamRoot - uamInterface = getUamRoot() - userNames: Dict[str, str] = {} - for row in result: - uid = row.get("createdByUserId") - if uid and uid not in userNames: - try: - user = uamInterface.getUser(uid) - userNames[uid] = user.get("username", uid[:8]) if user else uid[:8] - except Exception: - userNames[uid] = uid[:8] - row["userName"] = userNames.get(uid, "") if uid else "" - except Exception: - for row in result: - row["userName"] = row.get("createdByUserId", "")[:8] if row.get("createdByUserId") else "" - + _attachCreatedByUserNamesToTransactionRows(result) return result @@ -1641,7 +1600,7 @@ def getMandateViewTransactions( aicoreProvider=t.get("aicoreProvider"), aicoreModel=t.get("aicoreModel"), createdByUserId=t.get("createdByUserId"), - createdAt=t.get("_createdAt"), + createdAt=t.get("sysCreatedAt"), mandateId=t.get("mandateId"), mandateName=t.get("mandateName") )) @@ -1796,7 +1755,7 @@ def getUserViewStatistics( skippedNotDebit = 0 for t in allTransactions: - createdAt = t.get("_createdAt") + createdAt = t.get("sysCreatedAt") if not createdAt: skippedNoDate += 1 continue @@ -1972,7 +1931,7 @@ def getUserViewTransactions( "aicoreProvider": t.get("aicoreProvider"), "aicoreModel": t.get("aicoreModel"), "createdByUserId": t.get("createdByUserId"), - "createdAt": t.get("_createdAt"), + "createdAt": t.get("sysCreatedAt"), "mandateId": t.get("mandateId"), "mandateName": t.get("mandateName"), "userId": t.get("userId"), @@ -2069,7 +2028,7 @@ def getUserViewTransactionsFilterValues( "aicoreProvider": t.get("aicoreProvider"), "aicoreModel": t.get("aicoreModel"), "createdByUserId": t.get("createdByUserId"), - "createdAt": t.get("_createdAt"), + "createdAt": t.get("sysCreatedAt"), "mandateId": t.get("mandateId"), "mandateName": t.get("mandateName"), "userId": t.get("userId"), diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 999d07df..17e0ef56 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -1,6 +1,6 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Form, Path, Request, status, Query, Response, Body +from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Form, Path, Request, status, Query, Response, Body, BackgroundTasks from fastapi.responses import JSONResponse from typing import List, Dict, Any, Optional import logging @@ -8,6 +8,7 @@ import json # Import auth module from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext +from modules.auth.authentication import _hasSysAdminRole # Import interfaces import modules.interfaces.interfaceDbManagement as interfaceDbManagement @@ -40,13 +41,16 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user): file_meta = mgmtInterface.getFile(fileId) feature_instance_id = "" mandate_id = "" + file_scope = "personal" if file_meta: if isinstance(file_meta, dict): feature_instance_id = file_meta.get("featureInstanceId") or "" mandate_id = file_meta.get("mandateId") or "" + file_scope = file_meta.get("scope") or "personal" else: feature_instance_id = getattr(file_meta, "featureInstanceId", None) or "" mandate_id = getattr(file_meta, "mandateId", None) or "" + file_scope = getattr(file_meta, "scope", None) or "personal" logger.info(f"Auto-index starting for {fileName} ({len(rawBytes)} bytes, {mimeType})") @@ -60,6 +64,7 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user): userId=userId, featureInstanceId=str(feature_instance_id) if feature_instance_id else "", mandateId=str(mandate_id) if mandate_id else "", + scope=file_scope, ) logger.info( f"Pre-scan complete for {fileName}: " @@ -265,7 +270,7 @@ def get_file_filter_values( pass try: - recordFilter = {"_createdBy": managementInterface.userId} + recordFilter = {"sysCreatedBy": managementInterface.userId} values = managementInterface.db.getDistinctColumnValues( FileItem, column, crossFilterPagination, recordFilter ) @@ -660,6 +665,145 @@ def batch_move_items( raise HTTPException(status_code=500, detail=str(e)) +# ── Scope & neutralize tagging endpoints (before /{fileId} catch-all) ───────── + +@router.patch("/{fileId}/scope") +@limiter.limit("30/minute") +def updateFileScope( + request: Request, + background_tasks: BackgroundTasks, + fileId: str = Path(..., description="ID of the file"), + scope: str = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Update the scope of a file. Global scope requires sysAdmin.""" + try: + validScopes = {"personal", "featureInstance", "mandate", "global"} + if scope not in validScopes: + raise HTTPException(status_code=400, detail=f"Invalid scope: {scope}. Must be one of {validScopes}") + + if scope == "global" and not context.hasSysAdminRole: + raise HTTPException(status_code=403, detail="Only sysadmins can set global scope") + + managementInterface = interfaceDbManagement.getInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + + managementInterface.updateFile(fileId, {"scope": scope}) + + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface + from modules.datamodels.datamodelKnowledge import FileContentIndex + knowledgeDb = getKnowledgeInterface() + indices = knowledgeDb.db.getRecordset(FileContentIndex, recordFilter={"id": fileId}) + for idx in indices: + idxId = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if idxId: + knowledgeDb.db.recordModify(FileContentIndex, idxId, {"scope": scope}) + except Exception as e: + logger.warning(f"Failed to update FileContentIndex scope for file {fileId}: {e}") + + fileMeta = managementInterface.getFile(fileId) + if fileMeta: + fn = fileMeta.fileName if hasattr(fileMeta, "fileName") else fileMeta.get("fileName", "") + mt = fileMeta.mimeType if hasattr(fileMeta, "mimeType") else fileMeta.get("mimeType", "") + + async def _runReindexAfterScopeChange(): + try: + await _autoIndexFile(fileId=fileId, fileName=fn, mimeType=mt, user=context.user) + except Exception as ex: + logger.warning("Re-index after scope change failed for %s: %s", fileId, ex) + + background_tasks.add_task(_runReindexAfterScopeChange) + + return {"fileId": fileId, "scope": scope, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating file scope: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch("/{fileId}/neutralize") +@limiter.limit("30/minute") +def updateFileNeutralize( + request: Request, + background_tasks: BackgroundTasks, + fileId: str = Path(..., description="ID of the file"), + neutralize: bool = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Toggle neutralization flag on a file. + + FAILSAFE: When turning neutralize ON, the existing Knowledge Store index + and all content chunks are deleted SYNCHRONOUSLY before the response is + returned. The re-index happens in a background task. If re-indexing + fails the file simply has no index — no un-neutralized data can leak. + """ + try: + managementInterface = interfaceDbManagement.getInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + + managementInterface.updateFile(fileId, {"neutralize": neutralize}) + + from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface + knowledgeDb = getKnowledgeInterface() + + if neutralize: + # ── CRITICAL: purge existing (potentially un-neutralized) index + # This MUST succeed before the response is sent so that no stale + # raw-text chunks remain searchable while re-indexing runs. + try: + knowledgeDb.deleteFileContentIndex(fileId) + logger.info("Neutralize toggle ON: deleted index + chunks for file %s", fileId) + except Exception as e: + logger.error("Neutralize toggle ON: FAILED to delete index for file %s: %s", fileId, e) + raise HTTPException( + status_code=500, + detail=f"Could not purge existing index for neutralization — aborting toggle. Error: {e}", + ) + else: + # Turning neutralize OFF: update metadata only; re-index will overwrite + try: + from modules.datamodels.datamodelKnowledge import FileContentIndex + indices = knowledgeDb.db.getRecordset(FileContentIndex, recordFilter={"id": fileId}) + for idx in indices: + idxId = idx.get("id") if isinstance(idx, dict) else getattr(idx, "id", None) + if idxId: + knowledgeDb.db.recordModify(FileContentIndex, idxId, { + "neutralizationStatus": "original", + "isNeutralized": False, + }) + except Exception as e: + logger.warning("Failed to update FileContentIndex after neutralize-OFF for %s: %s", fileId, e) + + # Background re-index (safe: if it fails, there is simply no index) + fileMeta = managementInterface.getFile(fileId) + if fileMeta: + fn = fileMeta.fileName if hasattr(fileMeta, "fileName") else fileMeta.get("fileName", "") + mt = fileMeta.mimeType if hasattr(fileMeta, "mimeType") else fileMeta.get("mimeType", "") + + async def _runReindexAfterNeutralizeToggle(): + try: + await _autoIndexFile(fileId=fileId, fileName=fn, mimeType=mt, user=context.user) + except Exception as ex: + logger.error("Re-index after neutralize toggle failed for %s: %s (file has NO index until next re-index)", fileId, ex) + + background_tasks.add_task(_runReindexAfterNeutralizeToggle) + + return {"fileId": fileId, "neutralize": neutralize, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating file neutralize flag: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # ── File endpoints with path parameters (catch-all /{fileId}) ───────────────── @router.get("/{fileId}", response_model=FileItem) @@ -728,6 +872,12 @@ def update_file( detail=f"File with ID {fileId} not found" ) + if file_info.get("scope") == "global" and not _hasSysAdminRole(str(currentUser.id)): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Only sysadmins can set global scope", + ) + # Check if user has access to the file using RBAC if not managementInterface.checkRbacPermission(FileItem, "update", fileId): raise HTTPException( diff --git a/modules/routes/routeDataSources.py b/modules/routes/routeDataSources.py new file mode 100644 index 00000000..e210d094 --- /dev/null +++ b/modules/routes/routeDataSources.py @@ -0,0 +1,97 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""PATCH endpoints for DataSource and FeatureDataSource scope/neutralize tagging.""" + +import logging +from typing import Any, Dict + +from fastapi import APIRouter, HTTPException, Depends, Path, Request, Body +from modules.auth import limiter, getRequestContext, RequestContext +from modules.auth.authentication import _hasSysAdminRole +from modules.datamodels.datamodelDataSource import DataSource +from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/datasources", + tags=["Data Sources"], + responses={ + 404: {"description": "Not found"}, + 400: {"description": "Bad request"}, + 401: {"description": "Unauthorized"}, + 403: {"description": "Forbidden"}, + 500: {"description": "Internal server error"}, + }, +) + +_VALID_SCOPES = {"personal", "featureInstance", "mandate", "global"} + + +def _findSourceRecord(db, sourceId: str): + """Look up a source by ID, checking DataSource first, then FeatureDataSource.""" + rec = db.getRecord(DataSource, sourceId) + if rec: + return rec, DataSource + rec = db.getRecord(FeatureDataSource, sourceId) + if rec: + return rec, FeatureDataSource + return None, None + + +@router.patch("/{sourceId}/scope") +@limiter.limit("30/minute") +def _updateDataSourceScope( + request: Request, + sourceId: str = Path(..., description="ID of the DataSource or FeatureDataSource"), + scope: str = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Update the scope of a DataSource or FeatureDataSource. Global scope requires sysAdmin.""" + if scope not in _VALID_SCOPES: + raise HTTPException(status_code=400, detail=f"Invalid scope: {scope}. Must be one of {_VALID_SCOPES}") + + if scope == "global" and not _hasSysAdminRole(context.user): + raise HTTPException(status_code=403, detail="Only sysadmins can set global scope") + + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rec, model = _findSourceRecord(rootIf.db, sourceId) + if not rec: + raise HTTPException(status_code=404, detail=f"DataSource {sourceId} not found") + + rootIf.db.recordModify(model, sourceId, {"scope": scope}) + logger.info("Updated scope=%s for %s %s", scope, model.__name__, sourceId) + return {"sourceId": sourceId, "scope": scope, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error("Error updating datasource scope: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch("/{sourceId}/neutralize") +@limiter.limit("30/minute") +def _updateDataSourceNeutralize( + request: Request, + sourceId: str = Path(..., description="ID of the DataSource or FeatureDataSource"), + neutralize: bool = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Toggle the neutralization flag on a DataSource or FeatureDataSource.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rec, model = _findSourceRecord(rootIf.db, sourceId) + if not rec: + raise HTTPException(status_code=404, detail=f"DataSource {sourceId} not found") + + rootIf.db.recordModify(model, sourceId, {"neutralize": neutralize}) + logger.info("Updated neutralize=%s for %s %s", neutralize, model.__name__, sourceId) + return {"sourceId": sourceId, "neutralize": neutralize, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error("Error updating datasource neutralize: %s", e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 7e903466..7cce66ca 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -639,14 +639,17 @@ def create_user( # MULTI-TENANT: Add user to current mandate via UserMandate with default "user" role if context.mandateId: - # Get "user" role ID userRole = appInterface.getRoleByLabel("user") - roleIds = [str(userRole.id)] if userRole else [] + if not userRole: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="No 'user' role found in system — cannot assign user to mandate" + ) appInterface.createUserMandate( userId=str(newUser.id), mandateId=str(context.mandateId), - roleIds=roleIds + roleIds=[str(userRole.id)] ) logger.info(f"Created UserMandate for user {newUser.id} in mandate {context.mandateId}") @@ -917,30 +920,29 @@ def send_password_link( expiryHours = int(APP_CONFIG.get("Auth_RESET_TOKEN_EXPIRY_HOURS", "24")) try: - from modules.serviceHub import Services - services = Services(targetUser) - + from modules.routes.routeSecurityLocal import _buildAuthEmailHtml, _sendAuthEmail + emailSubject = "PowerOn - Passwort setzen" - emailBody = f""" -Hallo {targetUser.fullName or targetUser.username}, + emailHtml = _buildAuthEmailHtml( + greeting=f"Hallo {targetUser.fullName or targetUser.username}", + bodyLines=[ + "Ein Administrator hat einen Link zum Setzen Ihres Passworts angefordert.", + "", + f"Ihr Benutzername: {targetUser.username}", + "", + "Klicken Sie auf die Schaltfläche, um Ihr Passwort zu setzen:", + ], + buttonText="Passwort setzen", + buttonUrl=magicLink, + footerText=f"Dieser Link ist {expiryHours} Stunden gültig. Falls Sie diese Anforderung nicht erwartet haben, kontaktieren Sie bitte Ihren Administrator.", + ) -Ein Administrator hat einen Link zum Setzen Ihres Passworts angefordert. - -Ihr Benutzername: {targetUser.username} - -Klicken Sie auf den folgenden Link, um Ihr Passwort zu setzen: -{magicLink} - -Dieser Link ist {expiryHours} Stunden gültig. - -Falls Sie diese Anforderung nicht erwartet haben, kontaktieren Sie bitte Ihren Administrator. -""" - - emailSent = services.messaging.sendEmailDirect( + emailSent = _sendAuthEmail( recipient=targetUser.email, subject=emailSubject, - message=emailBody, - userId=str(targetUser.id) + message="", + userId=str(targetUser.id), + htmlOverride=emailHtml, ) if not emailSent: diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index cb913137..ccefcc87 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -292,37 +292,24 @@ def create_invitation( emailConnector = ConnectorMessagingEmail() if instance_label: emailSubject = f"Einladung zur Feature-Instanz {instance_label}" - invite_text = f"der Feature-Instanz {instance_label} (Mandant: {mandateName}) beizutreten" + invite_desc = f"der Feature-Instanz «{instance_label}» (Mandant: {mandateName}) beizutreten" else: emailSubject = f"Einladung zu {mandateName}" - invite_text = f"dem Mandanten {mandateName} beizutreten" - emailBody = f""" - -
-Hallo {display_name},
-Sie wurden eingeladen, {invite_text}.
-Klicken Sie auf den folgenden Link, um die Einladung anzunehmen:
- -
- Oder kopieren Sie diesen Link in Ihren Browser:
- {inviteUrl}
-
- Diese Einladung ist {data.expiresInHours} Stunden gültig. -
-- Diese E-Mail wurde automatisch von PowerOn gesendet. -
- - - """ - + invite_desc = f"dem Mandanten «{mandateName}» beizutreten" + + from modules.routes.routeSecurityLocal import _buildAuthEmailHtml + emailBody = _buildAuthEmailHtml( + greeting=f"Hallo {display_name}", + bodyLines=[ + f"Sie wurden eingeladen, {invite_desc}.", + "", + "Klicken Sie auf die Schaltfläche, um die Einladung anzunehmen:", + ], + buttonText="Einladung annehmen", + buttonUrl=inviteUrl, + footerText=f"Diese Einladung ist {data.expiresInHours} Stunden gültig.", + ) + emailConnector.send( recipient=email_val, subject=emailSubject, @@ -376,6 +363,8 @@ def create_invitation( f"to {target_desc}, expires in {data.expiresInHours}h" ) + # Invitation extends PowerOnModel: recordCreate/_saveRecord set sysCreatedAt and sysCreatedBy automatically. + # API response uses createdAt/createdBy; map from the system fields (no separate createdAt column on model). return InvitationResponse( id=str(createdRecord.get("id")), token=str(createdRecord.get("token")), @@ -384,8 +373,8 @@ def create_invitation( roleIds=createdRecord.get("roleIds", []), targetUsername=createdRecord.get("targetUsername"), email=createdRecord.get("email"), - createdBy=str(createdRecord.get("createdBy")), - createdAt=createdRecord.get("createdAt"), + createdBy=str(createdRecord["sysCreatedBy"]), + createdAt=float(createdRecord["sysCreatedAt"]), expiresAt=createdRecord.get("expiresAt"), usedBy=createdRecord.get("usedBy"), usedAt=createdRecord.get("usedAt"), diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index ff775ec3..2b380db0 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -219,6 +219,7 @@ async def auth_login_callback( user_info = user_info_response.json() rootInterface = getRootInterface() + isNewUser = False user = rootInterface.getUserByUsername(user_info.get("email")) if not user: user = rootInterface.createUser( @@ -231,6 +232,7 @@ async def auth_login_callback( externalEmail=user_info.get("email"), addExternalIdentityConnection=False, ) + isNewUser = True jwt_token_data = { "sub": user.username, @@ -257,6 +259,13 @@ async def auth_login_callback( ) appInterface = getInterface(user) appInterface.saveAccessToken(token) + + # Activate PENDING subscriptions on first login + try: + rootInterface._activatePendingSubscriptions(str(user.id)) + except Exception as subErr: + logger.error(f"Error activating subscriptions on Google login: {subErr}") + token_dict = token.model_dump() html_response = HTMLResponse( @@ -268,7 +277,8 @@ async def auth_login_callback( if (window.opener) {{ window.opener.postMessage({{ type: 'google_auth_success', - token_data: {json.dumps(token_dict)} + token_data: {json.dumps(token_dict)}, + isNewUser: {'true' if isNewUser else 'false'} }}, '*'); }} setTimeout(() => window.close(), 1000); diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 19c8f8f7..9ec4fc38 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -4,7 +4,7 @@ Routes for local security and authentication. """ -from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body +from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body, Path from fastapi.security import OAuth2PasswordRequestForm import logging from typing import Dict, Any @@ -14,7 +14,7 @@ import uuid from jose import jwt # Import auth modules -from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM +from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM, getRequestContext, RequestContext from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate @@ -26,36 +26,122 @@ from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) -def _sendAuthEmail(recipient: str, subject: str, message: str, userId: str = None) -> bool: +def _buildAuthEmailHtml( + greeting: str, + bodyLines: list, + buttonText: str = None, + buttonUrl: str = None, + footerText: str = None, +) -> str: + """Build a branded HTML email for authentication flows. + + Uses the same visual design as notifyMandateAdmins._renderHtmlEmail + (dark header, clean body, operator footer). + """ + import html as _html + + paragraphsHtml = "" + for line in bodyLines: + if line == "": + paragraphsHtml += '\n' + else: + escaped = _html.escape(str(line)) + paragraphsHtml += f'
{escaped}
\n' + + buttonBlock = "" + if buttonText and buttonUrl: + buttonBlock = f''' ++ {_html.escape(buttonUrl)} +
''' + + footerNote = "" + if footerText: + footerNote = f'{_html.escape(footerText)}
\n' + + operatorLine = "" + try: + from modules.shared.configuration import APP_CONFIG + parts = [p for p in [ + APP_CONFIG.get("Operator_CompanyName", ""), + APP_CONFIG.get("Operator_Address", ""), + APP_CONFIG.get("Operator_VatNumber", ""), + ] if p] + if parts: + operatorLine = ( + f'' + f'{_html.escape(" | ".join(parts))}
\n' + ) + except Exception: + pass + + return f''' + + + +| + + |