From 19be818fbb360a6a3a0d8378781a8beb7976dba0 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 17 Apr 2026 11:50:24 +0200 Subject: [PATCH] testing fixes, udb source handling fixes --- modules/datamodels/datamodelBilling.py | 2 +- modules/datamodels/datamodelFileFolder.py | 16 ++++++++ modules/interfaces/interfaceBootstrap.py | 9 ++++- modules/routes/routeDataFiles.py | 39 +++++++++++++++++++ modules/routes/routeDataMandates.py | 34 +++++++++++++--- modules/routes/routeDataSources.py | 31 ++++++++++++++- modules/routes/routeDataUsers.py | 5 ++- modules/system/mainSystem.py | 5 +++ .../workflows/automation2/executionEngine.py | 13 ++++++- .../executors/actionNodeExecutor.py | 4 ++ .../methods/methodAi/actions/consolidate.py | 4 ++ .../methods/methodAi/actions/generateCode.py | 4 ++ .../methodAi/actions/generateDocument.py | 4 ++ .../methods/methodAi/actions/process.py | 9 +++++ .../methods/methodAi/actions/webResearch.py | 9 +++++ .../methodTrustee/actions/extractFromFiles.py | 4 ++ 16 files changed, 179 insertions(+), 13 deletions(-) diff --git a/modules/datamodels/datamodelBilling.py b/modules/datamodels/datamodelBilling.py index f662e28c..8718413c 100644 --- a/modules/datamodels/datamodelBilling.py +++ b/modules/datamodels/datamodelBilling.py @@ -138,7 +138,7 @@ class BillingSettings(BaseModel): warningThresholdPercent: float = Field( default=10.0, - description="Warning threshold as percentage", + description="Benachrichtigung wenn das AI-Guthaben unter diesen Prozentsatz des Gesamtbudgets fällt", json_schema_extra={"label": "Warnschwelle (%)"}, ) diff --git a/modules/datamodels/datamodelFileFolder.py b/modules/datamodels/datamodelFileFolder.py index e3b0ba1a..e3d2ce87 100644 --- a/modules/datamodels/datamodelFileFolder.py +++ b/modules/datamodels/datamodelFileFolder.py @@ -54,6 +54,22 @@ class FileFolder(PowerOnModel): "fk_target": {"db": "poweron_app", "table": "FeatureInstance"}, }, ) + scope: str = Field( + default="personal", + description="Data visibility scope: personal, featureInstance, mandate, global. Inherited by files in this folder.", + json_schema_extra={ + "label": "Sichtbarkeit", + "frontend_type": "select", + "frontend_readonly": False, + "frontend_required": False, + "frontend_options": [ + {"value": "personal", "label": "Persönlich"}, + {"value": "featureInstance", "label": "Feature-Instanz"}, + {"value": "mandate", "label": "Mandant"}, + {"value": "global", "label": "Global"}, + ], + }, + ) neutralize: bool = Field( default=False, description="Whether files in this folder should be neutralized before AI processing. Inherited by new/moved files.", diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 8f6e75fc..707d9acc 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -1510,12 +1510,16 @@ def _ensureUiContextRules(db: DatabaseConnector) -> None: if roleId and item: existingCombinations.add((roleId, item)) - # Check each navigation item and add missing rules + # Check each navigation item and add missing rules (including subgroup items) missingRules = [] for section in NAVIGATION_SECTIONS: isAdminSection = section.get("adminOnly", False) - for item in section.get("items", []): + allItems = list(section.get("items", [])) + for subgroup in section.get("subgroups", []): + allItems.extend(subgroup.get("items", [])) + + for item in allItems: objectKey = item.get("objectKey") if not objectKey: continue @@ -1865,6 +1869,7 @@ def _createStoreResourceRules(db: DatabaseConnector) -> None: "resource.store.teamsbot", "resource.store.workspace", "resource.store.commcoach", + "resource.store.trustee", ] storeRules = [] diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index ebfd0e38..544f0085 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -502,6 +502,45 @@ def move_folder( raise HTTPException(status_code=500, detail=str(e)) +@router.patch("/folders/{folderId}/scope") +@limiter.limit("10/minute") +def _updateFolderScope( + request: Request, + folderId: str = Path(..., description="ID of the folder"), + scope: str = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Update the scope of a folder. Propagates to all files inside (recursively). Global scope requires sysAdmin.""" + validScopes = {"personal", "featureInstance", "mandate", "global"} + if scope not in validScopes: + raise HTTPException(status_code=400, detail=f"Invalid scope: {scope}. Must be one of {validScopes}") + if scope == "global" and not _hasSysAdminRole(context.user): + raise HTTPException(status_code=403, detail=routeApiMsg("Only sysadmins can set global scope")) + try: + mgmt = interfaceDbManagement.getInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, + ) + folder = mgmt.getFolder(folderId) + if not folder: + raise HTTPException(status_code=404, detail=routeApiMsg("Folder not found")) + mgmt.updateFolder(folderId, {"scope": scope}) + fileIds = _collectFolderFileIds(mgmt, folderId) + for fid in fileIds: + try: + mgmt.updateFile(fid, {"scope": scope}) + except Exception as e: + logger.error("Folder scope propagation: failed to update file %s: %s", fid, e) + logger.info("Updated scope=%s for folder %s: %d files affected", scope, folderId, len(fileIds)) + return {"folderId": folderId, "scope": scope, "filesUpdated": len(fileIds)} + except HTTPException: + raise + except Exception as e: + logger.error(f"Error updating folder scope: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.patch("/folders/{folderId}/neutralize") @limiter.limit("10/minute") def updateFolderNeutralize( diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index d2dfb2fb..9c48ccd1 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -17,7 +17,7 @@ import json from pydantic import BaseModel, Field # Import auth module -from modules.auth import limiter, requireSysAdminRole, getRequestContext, RequestContext +from modules.auth import limiter, requireSysAdminRole, getRequestContext, getCurrentUser, RequestContext # Import interfaces import modules.interfaces.interfaceDbApp as interfaceDbApp @@ -341,32 +341,54 @@ def create_mandate( detail=f"Failed to create mandate: {str(e)}" ) +_MANDATE_ADMIN_EDITABLE_FIELDS = {"label"} + @router.put("/{mandateId}", response_model=Mandate) @limiter.limit("10/minute") def update_mandate( request: Request, mandateId: str = Path(..., description="ID of the mandate to update"), mandateData: dict = Body(..., description="Mandate update data"), - currentUser: User = Depends(requireSysAdminRole) + currentUser: User = Depends(getCurrentUser) ) -> Mandate: """ Update an existing mandate. - MULTI-TENANT: SysAdmin-only. + MULTI-TENANT: + - SysAdmin: full update + - MandateAdmin: only label """ + from modules.auth import _hasSysAdminRole as _checkSysAdminRole + isSysAdmin = _checkSysAdminRole(str(currentUser.id)) + + if not isSysAdmin: + context = getRequestContext(request, currentUser=currentUser) + isMandateAdmin = _hasMandateAdminRole(context, mandateId) + if not isMandateAdmin: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("Admin role required to update mandate") + ) + try: logger.debug(f"Updating mandate {mandateId} with data: {mandateData}") appInterface = interfaceDbApp.getRootInterface() - # Check if mandate exists existingMandate = appInterface.getMandate(mandateId) if not existingMandate: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f"Mandate with ID {mandateId} not found" ) + + if not isSysAdmin: + mandateData = {k: v for k, v in mandateData.items() if k in _MANDATE_ADMIN_EDITABLE_FIELDS} + if not mandateData: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("No editable fields submitted") + ) - # Update mandate - mandateData is already a dict updatedMandate = appInterface.updateMandate(mandateId, mandateData) if not updatedMandate: @@ -375,7 +397,7 @@ def update_mandate( detail=routeApiMsg("Failed to update mandate") ) - logger.info(f"Mandate {mandateId} updated by SysAdmin {currentUser.id}") + logger.info(f"Mandate {mandateId} updated by user {currentUser.id} (sysadmin={isSysAdmin})") return updatedMandate except HTTPException: diff --git a/modules/routes/routeDataSources.py b/modules/routes/routeDataSources.py index db4b9a4f..03f6e8e3 100644 --- a/modules/routes/routeDataSources.py +++ b/modules/routes/routeDataSources.py @@ -3,7 +3,7 @@ """PATCH endpoints for DataSource and FeatureDataSource scope/neutralize tagging.""" import logging -from typing import Any, Dict +from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Depends, Path, Request, Body from modules.auth import limiter, getRequestContext, RequestContext @@ -97,3 +97,32 @@ def _updateDataSourceNeutralize( except Exception as e: logger.error("Error updating datasource neutralize: %s", e) raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch("/{sourceId}/neutralize-fields") +@limiter.limit("30/minute") +def _updateNeutralizeFields( + request: Request, + sourceId: str = Path(..., description="ID of the FeatureDataSource"), + neutralizeFields: List[str] = Body(..., embed=True), + context: RequestContext = Depends(getRequestContext), +) -> Dict[str, Any]: + """Update the list of field names to neutralize on a FeatureDataSource.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootIf = getRootInterface() + rec = rootIf.db.getRecord(FeatureDataSource, sourceId) + if not rec: + raise HTTPException(status_code=404, detail=f"FeatureDataSource {sourceId} not found") + + cleanFields = [f for f in neutralizeFields if f and isinstance(f, str)] if neutralizeFields else [] + rootIf.db.recordModify(FeatureDataSource, sourceId, { + "neutralizeFields": cleanFields if cleanFields else None, + }) + logger.info("Updated neutralizeFields=%s for FeatureDataSource %s", cleanFields, sourceId) + return {"sourceId": sourceId, "neutralizeFields": cleanFields, "updated": True} + except HTTPException: + raise + except Exception as e: + logger.error("Error updating neutralizeFields: %s", e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 92e9cb1f..bc32dfee 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -481,8 +481,9 @@ def update_user( detail=f"User with ID {userId} not found" ) - # Update user - updatedUser = rootInterface.updateUser(userId, userData) + # SysAdmins may toggle the isSysAdmin flag on other users + callerIsSysAdmin = context.isSysAdmin or context.hasSysAdminRole + updatedUser = rootInterface.updateUser(userId, userData, allowSysAdminChange=(callerIsSysAdmin and not isSelfUpdate)) if not updatedUser: raise HTTPException( diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index 59a42fcb..3eb0d981 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -505,6 +505,11 @@ RESOURCE_OBJECTS = [ "label": "Store: CommCoach", "meta": {"category": "store", "featureCode": "commcoach"} }, + { + "objectKey": "resource.store.trustee", + "label": "Store: Trustee", + "meta": {"category": "store", "featureCode": "trustee"} + }, { "objectKey": "resource.system.api.auth", "label": "Authentifizierungs-API", diff --git a/modules/workflows/automation2/executionEngine.py b/modules/workflows/automation2/executionEngine.py index d3a51800..92615062 100644 --- a/modules/workflows/automation2/executionEngine.py +++ b/modules/workflows/automation2/executionEngine.py @@ -28,6 +28,8 @@ from modules.workflows.automation2.executors import ( ) from modules.features.graphicalEditor.portTypes import _normalizeToSchema from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException as _SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError as _BillingContextError from modules.workflows.automation2.runEnvelope import normalize_run_envelope logger = logging.getLogger(__name__) @@ -279,7 +281,7 @@ async def _executeWithRetry(executor, node, context, maxRetries: int = 0, retryD try: result = await executor.execute(node, context) return result, attempt - except (PauseForHumanTaskError, PauseForEmailWaitError): + except (PauseForHumanTaskError, PauseForEmailWaitError, _SubscriptionInactiveException, _BillingContextError): raise except Exception as e: lastError = e @@ -488,6 +490,10 @@ async def executeGraph( _updateStepLog(automation2_interface, _rStepId, "completed", durationMs=int((time.time() - _rStepStart) * 1000)) raise + except (_SubscriptionInactiveException, _BillingContextError): + _updateStepLog(automation2_interface, _rStepId, "failed", + error="Subscription/Billing error", durationMs=int((time.time() - _rStepStart) * 1000)) + raise except Exception as ex: _updateStepLog(automation2_interface, _rStepId, "failed", error=str(ex), durationMs=int((time.time() - _rStepStart) * 1000)) @@ -625,6 +631,11 @@ async def executeGraph( _updateStepLog(automation2_interface, _bStepId, "completed", durationMs=int((time.time() - _bStepStart) * 1000)) raise + except (_SubscriptionInactiveException, _BillingContextError): + if _bStepId: + _updateStepLog(automation2_interface, _bStepId, "failed", + error="Subscription/Billing error", durationMs=int((time.time() - _bStepStart) * 1000)) + raise except Exception as ex: if _bStepId: _updateStepLog(automation2_interface, _bStepId, "failed", diff --git a/modules/workflows/automation2/executors/actionNodeExecutor.py b/modules/workflows/automation2/executors/actionNodeExecutor.py index e431e83f..31cfc39c 100644 --- a/modules/workflows/automation2/executors/actionNodeExecutor.py +++ b/modules/workflows/automation2/executors/actionNodeExecutor.py @@ -15,6 +15,8 @@ from modules.features.graphicalEditor.portTypes import ( _normalizeError, _unwrapTransit, ) +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException as _SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError as _BillingContextError logger = logging.getLogger(__name__) @@ -299,6 +301,8 @@ class ActionNodeExecutor: try: executor = ActionExecutor(self.services) result = await executor.executeAction(methodName, actionName, resolvedParams) + except (_SubscriptionInactiveException, _BillingContextError): + raise except Exception as e: logger.exception("ActionNodeExecutor node %s FAILED: %s", nodeId, e) return _normalizeError(e, outputSchema) diff --git a/modules/workflows/methods/methodAi/actions/consolidate.py b/modules/workflows/methods/methodAi/actions/consolidate.py index 7a7d7982..fa622507 100644 --- a/modules/workflows/methods/methodAi/actions/consolidate.py +++ b/modules/workflows/methods/methodAi/actions/consolidate.py @@ -7,6 +7,8 @@ from typing import Any, Dict, List from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum from modules.datamodels.datamodelChat import ActionResult +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError logger = logging.getLogger(__name__) @@ -66,6 +68,8 @@ async def consolidate(self, parameters: Dict[str, Any]) -> ActionResult: options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE), ) resp = await ai_service.callAi(req) + except (SubscriptionInactiveException, BillingContextError): + raise except Exception as e: logger.exception("consolidate: AI call failed: %s", e) return ActionResult.isFailure(error=str(e)) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index c616006b..313057a0 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -8,6 +8,8 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError logger = logging.getLogger(__name__) @@ -125,6 +127,8 @@ async def generateCode(self, parameters: Dict[str, Any]) -> ActionResult: return ActionResult.isSuccess(documents=documents) + except (SubscriptionInactiveException, BillingContextError): + raise except Exception as e: logger.error(f"Error in code generation: {str(e)}") return ActionResult.isFailure(error=str(e)) diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 8bb33f9d..0709b924 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -8,6 +8,8 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError logger = logging.getLogger(__name__) @@ -127,6 +129,8 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult: return ActionResult.isSuccess(documents=documents) + except (SubscriptionInactiveException, BillingContextError): + raise except Exception as e: logger.error(f"Error in document generation: {str(e)}") return ActionResult.isFailure(error=str(e)) diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index 0c893cb4..9332477d 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -10,6 +10,8 @@ from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError logger = logging.getLogger(__name__) @@ -340,6 +342,13 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult: return ActionResult.isSuccess(documents=final_documents) + except (SubscriptionInactiveException, BillingContextError): + try: + if operationId: + self.services.chat.progressLogFinish(operationId, False) + except Exception: + pass + raise except Exception as e: logger.error(f"Error in AI processing: {str(e)}") diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 0c3e3d5f..2c873396 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -8,6 +8,8 @@ import json from typing import Dict, Any from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.serviceCenter import ServiceCenterContext, getService, can_access_service +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError logger = logging.getLogger(__name__) @@ -112,6 +114,13 @@ async def webResearch(self, parameters: Dict[str, Any]) -> ActionResult: return ActionResult.isSuccess(documents=[actionDocument]) + except (SubscriptionInactiveException, BillingContextError): + try: + if operationId: + self.services.chat.progressLogFinish(operationId, False) + except Exception: + pass + raise except Exception as e: logger.error(f"Error in web research: {str(e)}") try: diff --git a/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py b/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py index 0502c6f6..37dd133d 100644 --- a/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py +++ b/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py @@ -18,6 +18,8 @@ from typing import Dict, Any, List, Optional, Tuple from modules.datamodels.datamodelChat import ActionResult, ActionDocument, ChatDocument, ChatMessage from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum +from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException as _SubscriptionInactiveException +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import BillingContextError as _BillingContextError logger = logging.getLogger(__name__) @@ -410,6 +412,8 @@ async def _extractOne( documentData=json.dumps(out), mimeType="application/json", ) + except (_SubscriptionInactiveException, _BillingContextError): + raise except Exception as e: logger.exception(f"Extract failed for {f.get('fileName')}") return ActionDocument(