From 7061dd4303afa7ca1cd9bf0082b75e45c09202ec Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 10 Jun 2026 16:32:45 +0200 Subject: [PATCH 1/8] ui generic rendering - base --- .../workspace/routeFeatureWorkspace.py | 48 ++++++--- .../interfaces/interfaceWorkflowAutomation.py | 8 +- modules/routes/routeWorkflowAutomation.py | 101 ++++++++++++------ modules/workflowAutomation/helpers.py | 13 +-- tests/demo/README.md | 36 +++---- tests/demo/test_demo_api.py | 16 ++- 6 files changed, 138 insertions(+), 84 deletions(-) diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index 66ed4966..df2603c4 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -1146,6 +1146,10 @@ async def getWorkspaceMessages( except Exception as e: logger.debug(f"getWorkspaceMessages: cannot read attachments for {workflowId}: {e}") + # Resolve labels server-side as the single source of truth: every returned + # ID carries a label, and IDs whose record no longer exists are dropped + # here. The client renders what it gets without any timing-dependent + # reconciliation against its own (asynchronously loaded) source lists. attachedDsLabels: Dict[str, str] = {} attachedFdsLabels: Dict[str, str] = {} if attachedDsIds or attachedFdsIds: @@ -1153,27 +1157,39 @@ async def getWorkspaceMessages( rootIf = getRootInterface() if attachedDsIds: from modules.datamodels.datamodelDataSource import DataSource + resolvedDsIds: List[str] = [] for dsId in attachedDsIds: try: records = rootIf.db.getRecordset(DataSource, recordFilter={"id": dsId}) - if records: - lbl = records[0].get("label") or records[0].get("path") or "" - if lbl: - attachedDsLabels[dsId] = str(lbl) - except Exception: - pass + except Exception as e: + # Transient DB error: keep the id; the client falls back to + # its own dataSources list for the label. + logger.warning(f"getWorkspaceMessages: label lookup failed for DataSource {dsId}: {e}") + resolvedDsIds.append(dsId) + continue + if not records: + continue # source was deleted -- drop the stale attachment + resolvedDsIds.append(dsId) + lbl = records[0].get("label") or records[0].get("path") or "" + attachedDsLabels[dsId] = str(lbl) if lbl else dsId + attachedDsIds = resolvedDsIds if attachedFdsIds: from modules.datamodels.datamodelFeatures import FeatureDataSource + resolvedFdsIds: List[str] = [] for fdsId in attachedFdsIds: try: records = rootIf.db.getRecordset(FeatureDataSource, recordFilter={"id": fdsId}) - if records: - tbl = records[0].get("tableName") or "" - lbl = records[0].get("label") or tbl - if lbl: - attachedFdsLabels[fdsId] = str(lbl) - except Exception: - pass + except Exception as e: + logger.warning(f"getWorkspaceMessages: label lookup failed for FeatureDataSource {fdsId}: {e}") + resolvedFdsIds.append(fdsId) + continue + if not records: + continue # source was deleted -- drop the stale attachment + resolvedFdsIds.append(fdsId) + tbl = records[0].get("tableName") or "" + lbl = records[0].get("label") or tbl + attachedFdsLabels[fdsId] = str(lbl) if lbl else fdsId + attachedFdsIds = resolvedFdsIds return JSONResponse({ "messages": items, @@ -1467,9 +1483,9 @@ async def listFeatureDataSources( from modules.serviceCenter.core.flagResolution import buildEffectiveByWorkspaceFds rootIf = getRootInterface() - recordFilter: dict = {} - if wsMandateId: - recordFilter["mandateId"] = wsMandateId + if not wsMandateId: + return JSONResponse({"featureDataSources": []}) + recordFilter: dict = {"mandateId": wsMandateId} records = rootIf.db.getRecordset(FeatureDataSource, recordFilter=recordFilter) or [] if not records: return JSONResponse({"featureDataSources": []}) diff --git a/modules/interfaces/interfaceWorkflowAutomation.py b/modules/interfaces/interfaceWorkflowAutomation.py index 84efe40a..c72c0b1e 100644 --- a/modules/interfaces/interfaceWorkflowAutomation.py +++ b/modules/interfaces/interfaceWorkflowAutomation.py @@ -375,11 +375,11 @@ class WorkflowAutomationObjects: return [] records = self.db.getRecordset( AutoRun, - recordFilter={}, + recordFilter={"workflowId": wf_ids}, ) if not records: return [] - runs = [dict(r) for r in records if r.get("workflowId") in wf_ids] + runs = [dict(r) for r in records] wf_by_id = {w["id"]: w for w in workflows} for r in runs: wf = wf_by_id.get(r.get("workflowId"), {}) @@ -652,7 +652,7 @@ class WorkflowAutomationObjects: def copyTemplateToUser(self, templateId: str) -> Optional[Dict[str, Any]]: """Copy a template to a new user-owned workflow with templateScope='user'.""" - template = self.getWorkflow(templateId) + template = self.db.getRecord(AutoWorkflow, templateId) if not template or not template.get("isTemplate"): return None data = { @@ -672,7 +672,7 @@ class WorkflowAutomationObjects: def shareTemplate(self, templateId: str, scope: str) -> Optional[Dict[str, Any]]: """Change a template's scope. Sets sharedReadOnly=True for shared scopes, False for user scope.""" - template = self.getWorkflow(templateId) + template = self.db.getRecord(AutoWorkflow, templateId) if not template or not template.get("isTemplate"): return None updated = self.db.recordModify(AutoWorkflow, templateId, { diff --git a/modules/routes/routeWorkflowAutomation.py b/modules/routes/routeWorkflowAutomation.py index fe3e8853..1ea2ba1b 100644 --- a/modules/routes/routeWorkflowAutomation.py +++ b/modules/routes/routeWorkflowAutomation.py @@ -69,12 +69,13 @@ async def _listWorkflows( try: db._ensureTableExists(AutoWorkflow) scopeFilter = _scopedWorkflowFilter(request) - if mandateId and scopeFilter is not None: - if mandateId not in (scopeFilter.get("mandateId") or []): + if mandateId: + scopeMandates = scopeFilter.get("mandateId") or [] + if isinstance(scopeMandates, str): + scopeMandates = [scopeMandates] + if mandateId not in scopeMandates: return {"items": [], "total": 0} scopeFilter = {"mandateId": mandateId} - elif mandateId and scopeFilter is None: - scopeFilter = {"mandateId": mandateId} params = _parsePaginationOr400(pagination) records = db.getRecordset(AutoWorkflow, recordFilter=scopeFilter) @@ -178,13 +179,13 @@ async def _listRuns( try: db._ensureTableExists(AutoRun) scopeFilter = _scopedRunFilter(request) - if mandateId: - if scopeFilter is None: - scopeFilter = {"mandateId": mandateId} - elif "mandateId" in scopeFilter: - if mandateId not in scopeFilter["mandateId"]: - return {"items": [], "total": 0} - scopeFilter = {"mandateId": mandateId} + if mandateId and scopeFilter and "mandateId" in scopeFilter: + scopeMandates = scopeFilter["mandateId"] + if isinstance(scopeMandates, str): + scopeMandates = [scopeMandates] + if mandateId not in scopeMandates: + return {"items": [], "total": 0} + scopeFilter = {"mandateId": mandateId} if workflowId: scopeFilter = {**(scopeFilter or {}), "workflowId": workflowId} @@ -242,16 +243,21 @@ async def _listTasks( db = _getWorkflowAutomationDb() try: db._ensureTableExists(AutoTask) - scopeFilter: Optional[Dict[str, Any]] = None + userId = str(request.user.id) if request.user else None + if not userId: + return {"items": [], "total": 0} + userMandateIds = _getUserMandateIds(userId) + if not userMandateIds: + return {"items": [], "total": 0} - if not request.isPlatformAdmin: - userId = str(request.user.id) if request.user else None - if not userId: - return {"items": [], "total": 0} - scopeFilter = {"assigneeId": userId} + wfFilter = {"mandateId": userMandateIds, "isTemplate": False} + db._ensureTableExists(AutoWorkflow) + wfs = db.getRecordset(AutoWorkflow, recordFilter=wfFilter) or [] + wfIds = [w.get("id") for w in wfs] + scopeFilter: Dict[str, Any] = {"workflowId": wfIds} if wfIds else {"workflowId": "__none__"} if status: - scopeFilter = {**(scopeFilter or {}), "status": status} + scopeFilter["status"] = status params = _parsePaginationOr400(pagination) records = db.getRecordset(AutoTask, recordFilter=scopeFilter) @@ -286,6 +292,27 @@ async def _listVersions( db.close() +# --------------------------------------------------------------------------- +# Editor chat history (stub — persistence not yet implemented) +# --------------------------------------------------------------------------- + +@router.get("/{workflowId}/chat/messages") +async def _getEditorChatMessages( + workflowId: str, + request: RequestContext = Depends(getRequestContext), +): + db = _getWorkflowAutomationDb() + try: + db._ensureTableExists(AutoWorkflow) + wf = db.getRecord(AutoWorkflow, workflowId) + if not wf: + raise HTTPException(status_code=404, detail="Workflow not found") + _validateWorkflowAccess(request, wf, "read") + return {"messages": []} + finally: + db.close() + + # --------------------------------------------------------------------------- # Step logs # --------------------------------------------------------------------------- @@ -424,9 +451,11 @@ def _listTemplates( raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required")) userId = str(context.user.id) - userMandateIds = _getUserMandateIds(userId) if not context.isPlatformAdmin else [] + userMandateIds = _getUserMandateIds(userId) + if mandateId and mandateId not in userMandateIds: + raise HTTPException(status_code=403, detail=routeApiMsg("Access denied")) effectiveMandateId = mandateId or (userMandateIds[0] if userMandateIds else None) - if not effectiveMandateId and not context.isPlatformAdmin: + if not effectiveMandateId: return {"templates": []} instanceId = None @@ -535,8 +564,10 @@ def _copyTemplate( mandateId = body.get("mandateId") if isinstance(body, dict) else None userId = str(context.user.id) + userMandateIds = _getUserMandateIds(userId) + if mandateId and mandateId not in userMandateIds: + raise HTTPException(status_code=403, detail=routeApiMsg("Access denied")) if not mandateId: - userMandateIds = _getUserMandateIds(userId) mandateId = userMandateIds[0] if userMandateIds else "" db = _getWorkflowAutomationDb() @@ -577,8 +608,10 @@ def _shareTemplate( mandateId = body.get("mandateId", "") userId = str(context.user.id) + userMandateIds = _getUserMandateIds(userId) + if mandateId and mandateId not in userMandateIds: + raise HTTPException(status_code=403, detail=routeApiMsg("Access denied")) if not mandateId: - userMandateIds = _getUserMandateIds(userId) mandateId = userMandateIds[0] if userMandateIds else "" db = _getWorkflowAutomationDb() @@ -984,14 +1017,12 @@ def _getMetrics( raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required")) userId = str(context.user.id) - userMandateIds = _getUserMandateIds(userId) if not context.isPlatformAdmin else [] + userMandateIds = _getUserMandateIds(userId) if mandateId: - if not context.isPlatformAdmin and mandateId not in userMandateIds: + if mandateId not in userMandateIds: raise HTTPException(status_code=403, detail=routeApiMsg("Access denied")) scopeFilter: Dict[str, Any] = {"mandateId": mandateId, "isTemplate": False} - elif context.isPlatformAdmin: - scopeFilter = {"isTemplate": False} elif userMandateIds: scopeFilter = {"mandateId": userMandateIds, "isTemplate": False} else: @@ -1001,13 +1032,21 @@ def _getMetrics( "totalTokens": 0, "totalCredits": 0.0, } + runScope = _scopedRunFilter(context) + if mandateId and runScope: + if "mandateId" in runScope: + if mandateId not in (runScope["mandateId"] if isinstance(runScope["mandateId"], list) else [runScope["mandateId"]]): + runScope = {"mandateId": "__none__"} + else: + runScope = {"mandateId": mandateId} + db = _getWorkflowAutomationDb() try: - workflows = db.getRecordset(AutoWorkflow, recordFilter=scopeFilter) or [] if db._ensureTableExists(AutoWorkflow) else [] - wfIds = [w.get("id") for w in workflows] - runFilter = {"workflowId": wfIds} if wfIds else {"workflowId": "__none__"} - runs = db.getRecordset(AutoRun, recordFilter=runFilter) or [] if db._ensureTableExists(AutoRun) else [] - tasks = db.getRecordset(AutoTask, recordFilter=runFilter) or [] if db._ensureTableExists(AutoTask) else [] + workflows = (db.getRecordset(AutoWorkflow, recordFilter=scopeFilter) or []) if db._ensureTableExists(AutoWorkflow) else [] + runs = (db.getRecordset(AutoRun, recordFilter=runScope) or []) if db._ensureTableExists(AutoRun) else [] + runIds = [r.get("id") for r in runs] + taskFilter = {"runId": runIds} if runIds else {"runId": "__none__"} + tasks = (db.getRecordset(AutoTask, recordFilter=taskFilter) or []) if db._ensureTableExists(AutoTask) else [] finally: db.close() diff --git a/modules/workflowAutomation/helpers.py b/modules/workflowAutomation/helpers.py index b2ba3f9f..9f28f274 100644 --- a/modules/workflowAutomation/helpers.py +++ b/modules/workflowAutomation/helpers.py @@ -126,9 +126,7 @@ def _isUserMandateAdmin(userId: str, mandateId: str) -> bool: def _scopedWorkflowFilter(context: RequestContext) -> Optional[Dict[str, Any]]: - """Build DB filter for listing workflows: mandate-scoped for members, None for sysadmin.""" - if context.isPlatformAdmin: - return None + """Build DB filter for listing workflows: always mandate-scoped by membership.""" userId = str(context.user.id) if context.user else None if not userId: return {"mandateId": "__impossible__"} @@ -139,14 +137,17 @@ def _scopedWorkflowFilter(context: RequestContext) -> Optional[Dict[str, Any]]: def _scopedRunFilter(context: RequestContext) -> Optional[Dict[str, Any]]: - """Build DB filter for listing runs: admin sees mandate runs, user sees own.""" - if context.isPlatformAdmin: - return None + """Build DB filter for listing runs: always mandate-scoped by membership. + Mandate admins see all runs in their mandates, regular members see own.""" userId = str(context.user.id) if context.user else None if not userId: return {"ownerId": "__impossible__"} mandateIds = _getUserMandateIds(userId) + if not mandateIds: + return {"ownerId": "__impossible__"} adminMandateIds = _getAdminMandateIds(userId, mandateIds) + if context.isPlatformAdmin: + return {"mandateId": mandateIds} if adminMandateIds: return {"mandateId": adminMandateIds} return {"ownerId": userId} diff --git a/tests/demo/README.md b/tests/demo/README.md index 9d68586f..b7ecd779 100644 --- a/tests/demo/README.md +++ b/tests/demo/README.md @@ -2,33 +2,33 @@ Automated tests for the investor demo configuration. -## Prerequisites +## SAFETY RULE (critical) -1. Gateway DB must be running and accessible -2. Demo config must be loaded first: Admin UI → `/admin/demo-config` → Load "Investor Demo April 2026" -3. RMA credentials must be set in `gateway/config.ini` +Tests in this suite MUST be strictly read-only towards the database. + +Pytest runs against the **real dev database** (there is no separate test DB). +Tests must NEVER load or remove demo configs — neither via direct module +calls (`cfg.load()` / `cfg.remove()`) nor via HTTP calls to +`/api/admin/demo-config//load|remove` (not even to assert 401/403). + +Background: on 2026-06-09 a demo test reloaded `investor-demo-2026` during a +pytest run. The demo mandates (HappyLife AG, Alpina Treuhand AG) were deleted +and recreated with new UUIDs, orphaning all real feature data (trustee +accounting incl. RMA connection, workflows, documents). The data had to be +recovered by remapping ~15'000 rows across 9 databases. + +Demo configs are loaded/removed manually via Admin UI → `/admin/demo-config`. ## Run ```bash -cd gateway/ - -# All demo tests (structural, no AI calls): +cd platform-core/ pytest tests/demo/ -v - -# Only bootstrap tests: -pytest tests/demo/test_demo_bootstrap.py -v - -# Only UC1 trustee: -pytest tests/demo/test_demo_uc1_trustee.py -v ``` ## Test files | File | What it tests | |------|--------------| -| `test_demo_bootstrap.py` | Idempotent load/remove, mandates, user, features, RMA, neutralization | -| `test_demo_uc1_trustee.py` | Trustee instances, RMA config, system workflow templates | -| `test_demo_uc2_realestate.py` | Workspace instances for agent demo | -| `test_demo_uc4_i18n.py` | i18n readiness, Spanish not pre-installed | -| `test_demo_neutralization.py` | Neutralization config enabled, test PDF exists | +| `test_demo_api.py` | Config auto-discovery (read-only), list endpoint rejects unauthenticated | +| `test_demo_data_files.py` | Demo data files exist in `demoData/` (filesystem only) | diff --git a/tests/demo/test_demo_api.py b/tests/demo/test_demo_api.py index 303b049d..b4a12ae6 100644 --- a/tests/demo/test_demo_api.py +++ b/tests/demo/test_demo_api.py @@ -44,7 +44,13 @@ class TestDemoConfigDiscovery: class TestDemoConfigApiEndpoints: - """Test API endpoints via TestClient.""" + """Test API endpoints via TestClient. + + SAFETY: Never call the load/remove endpoints here - not even to assert + 401/403. Tests run against the real dev database; if auth/CSRF ever lets + a request through, demo mandates get deleted and recreated with new UUIDs, + orphaning all real feature data (happened on 2026-06-09). + """ @pytest.fixture(scope="class") def client(self): @@ -55,11 +61,3 @@ class TestDemoConfigApiEndpoints: def test_listEndpointRejectsUnauthenticated(self, client): response = client.get("/api/admin/demo-config") assert response.status_code in (401, 403) - - def test_loadEndpointRejectsUnauthenticated(self, client): - response = client.post("/api/admin/demo-config/investor-demo-2026/load") - assert response.status_code in (401, 403) - - def test_removeEndpointRejectsUnauthenticated(self, client): - response = client.post("/api/admin/demo-config/investor-demo-2026/remove") - assert response.status_code in (401, 403) From 535bd431748609d2adec08dad7b3e26a127a26b3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 11 Jun 2026 16:47:21 +0200 Subject: [PATCH 2/8] model fixes --- app.py | 30 ++++++++++ modules/aicore/aicorePluginMistral.py | 3 +- modules/aicore/aicorePluginOpenai.py | 24 +------- modules/aicore/aicorePluginPrivateLlm.py | 52 ++++++++++++++-- modules/datamodels/datamodelKnowledge.py | 10 ++-- .../features/trustee/routeFeatureTrustee.py | 39 ++++++++---- modules/interfaces/interfaceDbKnowledge.py | 59 +++++++++++++++++++ modules/routes/routeDataFiles.py | 27 +++++++-- 8 files changed, 196 insertions(+), 48 deletions(-) diff --git a/app.py b/app.py index 20ad435c..1ce31ae5 100644 --- a/app.py +++ b/app.py @@ -176,6 +176,20 @@ def initLogging(): pass return True + # Suppress h11 LocalProtocolError ("Can't send data when our state is ERROR") + # from uvicorn when a client disconnects mid-response (browser abort, HMR, navigation). + # The asyncio event-loop handler (below) only catches event-loop-level exceptions; + # uvicorn logs this via the standard logging module before it reaches the event loop. + class ClientDisconnectFilter(logging.Filter): + def filter(self, record): + if record.exc_info: + excType = record.exc_info[0] + if excType and getattr(excType, "__name__", "") == "LocalProtocolError": + return False + if isinstance(record.msg, str) and "LocalProtocolError" in record.msg: + return False + return True + # Add filter to normalize problematic unicode (e.g., arrows) to ASCII for terminals like cp1252 class UnicodeArrowFilter(logging.Filter): def filter(self, record): @@ -204,6 +218,7 @@ def initLogging(): consoleHandler.addFilter(ChromeDevToolsFilter()) consoleHandler.addFilter(HttpcoreStarFilter()) consoleHandler.addFilter(HTTPDebugFilter()) + consoleHandler.addFilter(ClientDisconnectFilter()) consoleHandler.addFilter(EmojiFilter()) consoleHandler.addFilter(UnicodeArrowFilter()) handlers.append(consoleHandler) @@ -227,6 +242,7 @@ def initLogging(): fileHandler.addFilter(ChromeDevToolsFilter()) fileHandler.addFilter(HttpcoreStarFilter()) fileHandler.addFilter(HTTPDebugFilter()) + fileHandler.addFilter(ClientDisconnectFilter()) fileHandler.addFilter(EmojiFilter()) fileHandler.addFilter(UnicodeArrowFilter()) handlers.append(fileHandler) @@ -255,6 +271,12 @@ def initLogging(): for loggerName in noisyLoggers: logging.getLogger(loggerName).setLevel(logging.WARNING) + # Apply ClientDisconnectFilter to uvicorn's own logger so the + # h11 LocalProtocolError is suppressed regardless of handler setup. + _disconnectFilter = ClientDisconnectFilter() + for _uvName in ("uvicorn.error", "uvicorn"): + logging.getLogger(_uvName).addFilter(_disconnectFilter) + # Log the current logging configuration logger = logging.getLogger(__name__) logger.info(f"Logging initialized with level {logLevelName}") @@ -347,6 +369,14 @@ async def lifespan(app: FastAPI): except Exception as e: logger.warning(f"Bootstrap check failed (may already be initialized): {str(e)}") + # Migrate vector column dimensions (idempotent — safe on every startup) + try: + from modules.interfaces.interfaceDbKnowledge import migrateVectorDimensions + migrateVectorDimensions() + logger.info("Vector dimension migration check completed") + except Exception as e: + logger.warning(f"Vector dimension migration failed (non-critical): {e}") + # Register all feature definitions in RBAC catalog (for /api/features/ endpoint) try: from modules.security.rbacCatalog import getCatalogService diff --git a/modules/aicore/aicorePluginMistral.py b/modules/aicore/aicorePluginMistral.py index 8e32c67b..887bfda9 100644 --- a/modules/aicore/aicorePluginMistral.py +++ b/modules/aicore/aicorePluginMistral.py @@ -343,7 +343,8 @@ class AiMistral(BaseConnectorAi): content="", success=False, error="No embeddingInput provided" ) - payload = {"model": model.name, "input": texts} + from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS + payload = {"model": model.name, "input": texts, "output_dimension": KNOWLEDGE_EMBEDDING_DIMENSIONS} response = await self.httpClient.post(model.apiUrl, json=payload) if response.status_code != 200: diff --git a/modules/aicore/aicorePluginOpenai.py b/modules/aicore/aicorePluginOpenai.py index 3667d742..56d3c2c0 100644 --- a/modules/aicore/aicorePluginOpenai.py +++ b/modules/aicore/aicorePluginOpenai.py @@ -297,27 +297,6 @@ class AiOpenai(BaseConnectorAi): version="text-embedding-3-small", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00002 ), - AiModel( - name="text-embedding-3-large", - displayName="OpenAI Embedding Large", - connectorType="openai", - apiUrl="https://api.openai.com/v1/embeddings", - temperature=0.0, - maxTokens=0, - contextLength=8191, - costPer1kTokensInput=0.00013, # $0.13/M tokens - costPer1kTokensOutput=0.0, - speedRating=9, - qualityRating=10, - functionCall=self.callEmbedding, - priority=PriorityEnum.QUALITY, - processingMode=ProcessingModeEnum.ADVANCED, - operationTypes=createOperationTypeRatings( - (OperationTypeEnum.EMBEDDING, 10) - ), - version="text-embedding-3-large", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00013 - ), AiModel( name="gpt-image-1", displayName="OpenAI GPT Image", @@ -547,7 +526,8 @@ class AiOpenai(BaseConnectorAi): content="", success=False, error="No embeddingInput provided" ) - payload = {"model": model.name, "input": texts} + from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS + payload = {"model": model.name, "input": texts, "dimensions": KNOWLEDGE_EMBEDDING_DIMENSIONS} response = await self.httpClient.post(model.apiUrl, json=payload) if response.status_code != 200: diff --git a/modules/aicore/aicorePluginPrivateLlm.py b/modules/aicore/aicorePluginPrivateLlm.py index b96a1c4a..598b7ffa 100644 --- a/modules/aicore/aicorePluginPrivateLlm.py +++ b/modules/aicore/aicorePluginPrivateLlm.py @@ -14,7 +14,7 @@ Models (current — L4 24 GB): Models (next-gen — RTX PRO 6000 96 GB, auto-activated when pulled in Ollama): - poweron-text-reasoning: Reasoning (deepseek-r1:70b); complex logic, math, planning - poweron-vision-general: Vision (llama4:scout); multimodal, long-context documents -- poweron-embed: Embedding (nomic-embed-text); local RAG embedding +- poweron-embed: Embedding (mxbai-embed-large); local RAG embedding (1024 dim) Pricing: byte-based (~per-token via bytes/4), configured via the PRICE_* constants below. """ @@ -377,7 +377,7 @@ class AiPrivateLlm(BaseConnectorAi): ), "ollamaModel": "llama4:scout" }, - # Local Embedding (nomic-embed-text — replaces OpenAI text-embedding-3-small) + # Local Embedding (mxbai-embed-large — nativ 1024 dim, MTEB 64.68) { "model": AiModel( name="poweron-embed", @@ -386,21 +386,21 @@ class AiPrivateLlm(BaseConnectorAi): apiUrl=f"{self.baseUrl}/v1/embeddings", temperature=0.0, maxTokens=0, - contextLength=8192, + contextLength=512, costPer1kTokensInput=PRICE_EMBED_PER_1K, costPer1kTokensOutput=0.0, speedRating=10, qualityRating=8, - functionCall=self.callAiText, + functionCall=self.callEmbedding, priority=PriorityEnum.COST, processingMode=ProcessingModeEnum.BASIC, operationTypes=createOperationTypeRatings( (OperationTypeEnum.EMBEDDING, 9), ), - version="nomic-embed-text", + version="mxbai-embed-large", calculatepriceCHF=_calcPrivateEmbedPriceCHF ), - "ollamaModel": "nomic-embed-text" + "ollamaModel": "mxbai-embed-large" }, ] @@ -505,6 +505,46 @@ class AiPrivateLlm(BaseConnectorAi): logger.error(f"Error calling Private-LLM text API: {str(e)}") raise HTTPException(status_code=500, detail=f"Error calling Private-LLM API: {str(e)}") + async def callEmbedding(self, modelCall: AiModelCall) -> AiModelResponse: + """Generate embeddings via the Private-LLM Embedding endpoint (OpenAI-compatible).""" + try: + model = modelCall.model + texts = modelCall.embeddingInput or [] + if not texts: + return AiModelResponse( + content="", success=False, error="No embeddingInput provided" + ) + + payload = {"model": model.version, "input": texts} + response = await self.httpClient.post(model.apiUrl, json=payload) + + if response.status_code != 200: + errorMessage = f"Private-LLM Embedding API error: {response.status_code} - {response.text}" + if response.status_code == 429: + raise RateLimitExceededException(errorMessage) + raise HTTPException(status_code=500, detail=errorMessage) + + responseJson = response.json() + embeddings = [item["embedding"] for item in responseJson["data"]] + usage = responseJson.get("usage", {}) + + return AiModelResponse( + content="", + success=True, + modelId=model.name, + tokensUsed={ + "input": usage.get("prompt_tokens", 0), + "output": 0, + "total": usage.get("total_tokens", 0), + }, + metadata={"embeddings": embeddings}, + ) + except (RateLimitExceededException, HTTPException): + raise + except Exception as e: + logger.error(f"Error calling Private-LLM Embedding API: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error calling Private-LLM Embedding API: {str(e)}") + async def callAiVision(self, modelCall: AiModelCall) -> AiModelResponse: """ Call the Private-LLM API for vision-based analysis. diff --git a/modules/datamodels/datamodelKnowledge.py b/modules/datamodels/datamodelKnowledge.py index ad837ab1..7816135a 100644 --- a/modules/datamodels/datamodelKnowledge.py +++ b/modules/datamodels/datamodelKnowledge.py @@ -9,7 +9,7 @@ These models support the 3-tier RAG architecture: - Global Layer: scope=global (sysAdmin only) - Workflow Layer: workflowId-scoped (WorkflowMemory) -Vector fields use json_schema_extra={"db_type": "vector(1536)"} for pgvector. +Vector fields use json_schema_extra with db_type=vector(KNOWLEDGE_EMBEDDING_DIMENSIONS) for pgvector. """ from typing import Dict, Any, List, Optional @@ -19,6 +19,8 @@ from modules.shared.i18nRegistry import i18nModel from modules.shared.timeUtils import getUtcTimestamp import uuid +KNOWLEDGE_EMBEDDING_DIMENSIONS = 1024 + @i18nModel("Datei-Inhaltsindex") class FileContentIndex(PowerOnModel): @@ -163,7 +165,7 @@ class ContentChunk(PowerOnModel): embedding: Optional[List[float]] = Field( default=None, description="pgvector embedding (NOT NULL for text chunks)", - json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"}, + json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"}, ) @@ -210,7 +212,7 @@ class RoundMemory(PowerOnModel): embedding: Optional[List[float]] = Field( default=None, description="Embedding of summary for semantic retrieval", - json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"}, + json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"}, ) @@ -251,5 +253,5 @@ class WorkflowMemory(PowerOnModel): embedding: Optional[List[float]] = Field( default=None, description="Optional embedding for semantic lookup", - json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"}, + json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"}, ) diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 8b5ba94a..dbc96013 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -81,6 +81,13 @@ def _parsePagination(pagination: Optional[str]) -> Optional[PaginationParams]: return None +def _recordField(record, field: str, default=None): + """Read a field from a DB record dict or model instance.""" + if isinstance(record, dict): + return record.get(field, default) + return getattr(record, field, default) + + def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: """ Validate that the user has access to the feature instance. @@ -365,7 +372,7 @@ def get_contract_options( result = interface.getAllContracts(None) items = result.items if hasattr(result, 'items') else result - return [{"value": c.id, "label": c.label or c.name or c.id} for c in items] + return [{"value": _recordField(c, "id"), "label": _recordField(c, "label") or _recordField(c, "name") or _recordField(c, "id")} for c in items] @router.get("/{instanceId}/documents/options", response_model=List[Dict[str, Any]]) @@ -381,7 +388,14 @@ def get_document_options( result = interface.getAllDocuments(None) items = result.items if hasattr(result, 'items') else result # Include 'id' for FK resolution in tables - return [{"id": d.id, "value": d.id, "label": d.documentName or d.id} for d in items] + return [ + { + "id": _recordField(d, "id"), + "value": _recordField(d, "id"), + "label": _recordField(d, "documentName") or _recordField(d, "id"), + } + for d in items + ] @router.get("/{instanceId}/positions/options", response_model=List[Dict[str, Any]]) @@ -397,18 +411,21 @@ def get_position_options( result = interface.getAllPositions(None) items = result.items if hasattr(result, 'items') else result - def _makePositionLabel(p: TrusteePosition) -> str: + def _makePositionLabel(p) -> str: parts = [] - if p.valuta: - parts.append(datetime.fromtimestamp(p.valuta, tz=timezone.utc).strftime("%Y-%m-%d")) - if p.company: - parts.append(p.company[:30]) - if p.desc: - parts.append(p.desc[:30]) - return " - ".join(parts) if parts else p.id + valuta = _recordField(p, "valuta") + if valuta: + parts.append(datetime.fromtimestamp(valuta, tz=timezone.utc).strftime("%Y-%m-%d")) + company = _recordField(p, "company") + if company: + parts.append(str(company)[:30]) + desc = _recordField(p, "desc") + if desc: + parts.append(str(desc)[:30]) + return " - ".join(parts) if parts else _recordField(p, "id") # Include 'id' for FK resolution in tables - return [{"id": p.id, "value": p.id, "label": _makePositionLabel(p)} for p in items] + return [{"id": _recordField(p, "id"), "value": _recordField(p, "id"), "label": _makePositionLabel(p)} for p in items] # ============================================================================ diff --git a/modules/interfaces/interfaceDbKnowledge.py b/modules/interfaces/interfaceDbKnowledge.py index c52d999e..9c5a9bd3 100644 --- a/modules/interfaces/interfaceDbKnowledge.py +++ b/modules/interfaces/interfaceDbKnowledge.py @@ -732,3 +732,62 @@ def getInterface(currentUser: Optional[User] = None) -> KnowledgeObjects: interface.setUserContext(currentUser) return interface + + +def migrateVectorDimensions(): + """Idempotent boot migration: ensures all vector columns match KNOWLEDGE_EMBEDDING_DIMENSIONS. + + Checks the actual pgvector dimension via pg_attribute.atttypmod. + If it differs from the target, nulls existing embeddings and alters the column type. + Safe to call on every startup — skips when dimensions already match or table doesn't exist. + """ + from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS + targetDim = KNOWLEDGE_EMBEDDING_DIMENSIONS + + interface = getInterface() + db = interface.db + + vectorTables = [ + ("ContentChunk", "embedding"), + ("RoundMemory", "embedding"), + ("WorkflowMemory", "embedding"), + ] + + for table, col in vectorTables: + try: + with db.borrowConn() as conn: + with conn.cursor() as cursor: + cursor.execute( + "SELECT COUNT(*) FROM information_schema.tables " + "WHERE LOWER(table_name) = LOWER(%s) AND table_schema = 'public'", + (table,), + ) + if cursor.fetchone()["count"] == 0: + continue + + cursor.execute( + "SELECT a.atttypmod FROM pg_attribute a " + "JOIN pg_class c ON a.attrelid = c.oid " + "JOIN pg_namespace n ON c.relnamespace = n.oid " + "WHERE c.relname = %s AND a.attname = %s AND n.nspname = 'public'", + (table, col), + ) + row = cursor.fetchone() + if not row: + continue + + currentDim = row["atttypmod"] + if currentDim == targetDim: + continue + + logger.info( + "Migrating %s.%s from vector(%s) to vector(%s) — clearing existing embeddings", + table, col, currentDim, targetDim, + ) + cursor.execute(f'UPDATE "{table}" SET "{col}" = NULL WHERE "{col}" IS NOT NULL') + cursor.execute( + f'ALTER TABLE "{table}" ALTER COLUMN "{col}" TYPE vector({targetDim})' + ) + logger.info("Migration of %s.%s completed", table, col) + except Exception as e: + logger.error("Vector dimension migration failed for %s.%s: %s", table, col, e) diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 63189e53..41625d26 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -700,6 +700,7 @@ def get_files( pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), mode: Optional[str] = Query(None, description="'filterValues' for distinct column values, 'ids' for all filtered IDs"), column: Optional[str] = Query(None, description="Column key (required when mode=filterValues)"), + owner: Optional[str] = Query(None, description="'me' for own files, 'shared' for files from others"), currentUser: User = Depends(getCurrentUser), context: RequestContext = Depends(getRequestContext) ): @@ -756,6 +757,21 @@ def get_files( def _filesToDicts(fileItems): return [f.model_dump() if hasattr(f, "model_dump") else (dict(f) if not isinstance(f, dict) else f) for f in fileItems] + ownerRecordFilter = None + ownerExcludeOwnFiles = False + ownerNorm = (owner or "").strip().lower() + if ownerNorm == "me": + ownerRecordFilter = {"sysCreatedBy": managementInterface.userId} + elif ownerNorm == "shared": + ownerExcludeOwnFiles = True + + def _applyOwnerFilter(items): + """Post-filter for owner=shared: exclude files created by current user.""" + if not ownerExcludeOwnFiles: + return items + uid = managementInterface.userId + return [f for f in items if (f.get("sysCreatedBy") if isinstance(f, dict) else getattr(f, "sysCreatedBy", None)) != uid] + if mode == "groupSummary": if not pagination: raise HTTPException(status_code=400, detail="pagination required for groupSummary") @@ -794,10 +810,12 @@ def get_files( return handleIdsMode(managementInterface.db, FileItem, pagination, recordFilter) if not groupByLevels: - # No grouping: let DB handle pagination directly (fastest path) - result = managementInterface.getAllFiles(pagination=paginationParams) + result = managementInterface.getAllFiles( + pagination=paginationParams, + recordFilter=ownerRecordFilter, + ) if paginationParams and hasattr(result, 'items'): - enriched = enrichRowsWithFkLabels(_filesToDicts(result.items), FileItem, db=appInterface.db) + enriched = _applyOwnerFilter(enrichRowsWithFkLabels(_filesToDicts(result.items), FileItem, db=appInterface.db)) resp: dict = { "items": enriched, "pagination": PaginationMetadata( @@ -811,7 +829,8 @@ def get_files( } else: items = result if isinstance(result, list) else (result.items if hasattr(result, "items") else [result]) - resp = {"items": enrichRowsWithFkLabels(_filesToDicts(items), FileItem, db=appInterface.db), "pagination": None} + enriched = _applyOwnerFilter(enrichRowsWithFkLabels(_filesToDicts(items), FileItem, db=appInterface.db)) + resp = {"items": enriched, "pagination": None} if viewMeta: resp["appliedView"] = viewMeta.model_dump() return resp From a1d9c686042d390eb35df66ef555d95419532d7f Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 11 Jun 2026 21:26:32 +0200 Subject: [PATCH 3/8] ux cleanup --- modules/auth/trustedDeviceService.py | 171 ++++++++++++++++ modules/connectors/connectorDbPostgre.py | 1 + modules/connectors/connectorVoiceGoogle.py | 17 ++ modules/datamodels/datamodelSecurity.py | 37 ++++ .../workspace/routeFeatureWorkspace.py | 177 ++++++++++++++++- modules/interfaces/interfaceAiObjects.py | 76 ++++++- modules/interfaces/interfaceDbApp.py | 8 +- modules/routes/routeAdminSessions.py | 185 ++++++++++++++++++ modules/routes/routeMfa.py | 7 + modules/routes/routeSecurityLocal.py | 49 +++-- .../serviceKnowledge/mainServiceKnowledge.py | 26 ++- 11 files changed, 721 insertions(+), 33 deletions(-) create mode 100644 modules/auth/trustedDeviceService.py create mode 100644 modules/routes/routeAdminSessions.py diff --git a/modules/auth/trustedDeviceService.py b/modules/auth/trustedDeviceService.py new file mode 100644 index 00000000..a707e70a --- /dev/null +++ b/modules/auth/trustedDeviceService.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +""" +Trusted Device Service. + +After successful MFA verification a device can be marked as trusted for a +configurable duration (default 60 days). On subsequent logins from the same +device the MFA step is skipped. + +Cookie: ``mfa_trusted`` (httpOnly, Secure, SameSite policy from jwtService). +DB: ``TrustedDevice`` table in poweron_app. + +Regulatory basis: +- NIST SP 800-63B Section 5.2.8: Verifier MAY re-authenticate only after a + configurable period when a device is bound to the subscriber. +- Microsoft, Google, AWS implement identical patterns. +""" + +import logging +import secrets +from typing import Optional + +from fastapi import Request, Response + +from modules.shared.configuration import APP_CONFIG +from modules.shared.timeUtils import getUtcNow, getUtcTimestamp +from modules.datamodels.datamodelSecurity import TrustedDevice + +logger = logging.getLogger(__name__) + +_COOKIE_NAME = "mfa_trusted" +_DEFAULT_TRUST_DAYS = 60 +_TOKEN_BYTES = 32 + + +def _getTrustDurationDays() -> int: + raw = (APP_CONFIG.get("MFA_TRUST_DURATION_DAYS") or "").strip() + if raw.isdigit() and int(raw) > 0: + return int(raw) + return _DEFAULT_TRUST_DAYS + + +def createTrustedDevice(userId: str, request: Request, response: Response, db) -> str: + """Create a TrustedDevice entry and set the cookie on the response. + + Returns the device token (cookie value). + """ + from modules.auth.jwtService import _cookiePolicy + + trustDays = _getTrustDurationDays() + deviceToken = secrets.token_urlsafe(_TOKEN_BYTES) + + now = getUtcTimestamp() + trustedUntil = now + (trustDays * 86400) + + device = TrustedDevice( + id=deviceToken, + userId=userId, + trustedUntil=trustedUntil, + userAgent=(request.headers.get("user-agent") or "")[:512], + ipAddress=_getClientIp(request), + createdAt=now, + ) + + try: + db.recordCreate(TrustedDevice, device.model_dump()) + except Exception as e: + logger.error(f"Failed to persist TrustedDevice for userId={userId}: {e}") + return "" + + useSecure, samesite, _ = _cookiePolicy() + response.set_cookie( + key=_COOKIE_NAME, + value=deviceToken, + httponly=True, + secure=useSecure, + samesite=samesite, + path="/", + max_age=trustDays * 86400, + ) + + logger.info(f"Trusted device created for userId={userId}, valid {trustDays}d") + return deviceToken + + +def isTrustedDevice(request: Request, userId: str, db) -> bool: + """Check if the current request comes from a trusted device for the given user.""" + deviceToken = request.cookies.get(_COOKIE_NAME) + if not deviceToken: + return False + + try: + records = db.getRecordset( + TrustedDevice, + recordFilter={"id": deviceToken, "userId": userId}, + ) + if not records: + return False + + device = records[0] + trustedUntil = device.get("trustedUntil", 0) + if isinstance(trustedUntil, (int, float)) and trustedUntil > getUtcTimestamp(): + return True + + return False + except Exception as e: + logger.warning(f"Error checking trusted device for userId={userId}: {e}") + return False + + +def revokeTrustedDevices(userId: str, db) -> int: + """Revoke all trusted devices for a user. Returns count of deleted entries.""" + try: + records = db.getRecordset(TrustedDevice, recordFilter={"userId": userId}) + count = 0 + for rec in records: + db.recordDelete(TrustedDevice, rec["id"]) + count += 1 + if count: + logger.info(f"Revoked {count} trusted device(s) for userId={userId}") + return count + except Exception as e: + logger.error(f"Failed to revoke trusted devices for userId={userId}: {e}") + return 0 + + +def clearTrustedDeviceCookie(response: Response) -> None: + """Clear the mfa_trusted cookie.""" + from modules.auth.jwtService import _cookiePolicy + + useSecure, samesite, samesiteHeader = _cookiePolicy() + secure_flag = "; Secure" if useSecure else "" + response.headers.append( + "Set-Cookie", + f"{_COOKIE_NAME}=deleted; Path=/; Max-Age=0; Expires=Thu, 01 Jan 1970 00:00:00 GMT; HttpOnly{secure_flag}; SameSite={samesiteHeader}" + ) + response.delete_cookie( + key=_COOKIE_NAME, + path="/", + secure=useSecure, + httponly=True, + samesite=samesite, + ) + + +def cleanupExpiredDevices(db) -> int: + """Remove TrustedDevice entries past their trustedUntil. Returns deleted count.""" + try: + records = db.getRecordset(TrustedDevice, recordFilter={}) + now = getUtcTimestamp() + count = 0 + for rec in records: + if rec.get("trustedUntil", 0) < now: + db.recordDelete(TrustedDevice, rec["id"]) + count += 1 + if count: + logger.info(f"Cleaned up {count} expired trusted device(s)") + return count + except Exception as e: + logger.error(f"Error cleaning up expired trusted devices: {e}") + return 0 + + +def _getClientIp(request: Request) -> Optional[str]: + """Extract client IP from request (respects X-Forwarded-For).""" + forwarded = request.headers.get("x-forwarded-for") + if forwarded: + return forwarded.split(",")[0].strip() + if request.client: + return request.client.host + return None diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 11b406ad..115f25e4 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -871,6 +871,7 @@ class DatabaseConnector: ("jsonb", "TEXT"): "TEXT USING \"{col}\"::text", ("text", "DOUBLE PRECISION"): _TEXT_TO_DOUBLE, ("text", "INTEGER"): "INTEGER USING NULLIF(\"{col}\", '')::integer", + ("text", "BOOLEAN"): "BOOLEAN USING CASE WHEN \"{col}\" IN ('true', '1', 't', 'yes') THEN TRUE ELSE FALSE END", ("timestamp without time zone", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}" AT TIME ZONE \'UTC\')', ("timestamp with time zone", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}")', ("date", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}"::timestamp AT TIME ZONE \'UTC\')', diff --git a/modules/connectors/connectorVoiceGoogle.py b/modules/connectors/connectorVoiceGoogle.py index 590fd26b..68e9ef84 100644 --- a/modules/connectors/connectorVoiceGoogle.py +++ b/modules/connectors/connectorVoiceGoogle.py @@ -19,6 +19,21 @@ from modules.shared.voiceCatalog import getDefaultVoice logger = logging.getLogger(__name__) +_STT_LANGUAGE_MAP = { + "de-CH": "de-DE", + "de-AT": "de-DE", + "en-GB": "en-US", + "en-AU": "en-US", + "fr-CH": "fr-FR", + "it-CH": "it-IT", + "pt-BR": "pt-PT", +} + + +def _normalizeSttLanguage(language: str) -> str: + """Map regional language variants to codes supported by Google STT models.""" + return _STT_LANGUAGE_MAP.get(language, language) + def _buildPrimarySttRecognitionFields( *, @@ -116,6 +131,7 @@ class ConnectorGoogleSpeech: Returns: Dict containing transcribed text, confidence, and metadata """ + language = _normalizeSttLanguage(language) try: # Treat sampleRate=0 as unknown (invalid value from client) if sampleRate is not None and sampleRate <= 0: @@ -480,6 +496,7 @@ class ConnectorGoogleSpeech: Dicts with keys: isFinal, transcript, confidence, stabilityScore, audioDurationSec; optionally endOfSingleUtterance, reconnectRequired """ + language = _normalizeSttLanguage(language) STREAM_LIMIT_SEC = 290 streamStartTs = time.time() totalAudioBytes = 0 diff --git a/modules/datamodels/datamodelSecurity.py b/modules/datamodels/datamodelSecurity.py index 280fdc9e..e8cc148f 100644 --- a/modules/datamodels/datamodelSecurity.py +++ b/modules/datamodels/datamodelSecurity.py @@ -124,6 +124,43 @@ class Token(PowerOnModel): return data +@i18nModel("Vertrauenswuerdiges Geraet") +class TrustedDevice(PowerOnModel): + """A device trusted after successful MFA verification (skips MFA for configured duration).""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Random token stored as httpOnly cookie value", + json_schema_extra={"label": "ID"}, + ) + userId: str = Field( + ..., + description="User this trusted device belongs to", + json_schema_extra={"label": "Benutzer-ID", "fk_target": {"db": "poweron_app", "table": "UserInDB", "labelField": "username"}}, + ) + trustedUntil: float = Field( + ..., + description="UTC timestamp until which the device is trusted", + json_schema_extra={"label": "Vertrauenswuerdig bis", "frontend_type": "timestamp"}, + ) + userAgent: Optional[str] = Field( + default=None, + description="Browser user agent at time of trust grant", + json_schema_extra={"label": "User-Agent"}, + ) + ipAddress: Optional[str] = Field( + default=None, + description="IP address at time of trust grant", + json_schema_extra={"label": "IP-Adresse"}, + ) + createdAt: float = Field( + default_factory=getUtcTimestamp, + description="When the device was trusted", + json_schema_extra={"label": "Erstellt am", "frontend_type": "timestamp"}, + ) + + model_config = ConfigDict(use_enum_values=True) + + @i18nModel("Authentifizierungsereignis") class AuthEvent(PowerOnModel): """Authentication event for audit logging.""" diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index df2603c4..c6d143dd 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -489,6 +489,158 @@ def _collectPriorFileIds(chatInterface, workflowId: str) -> List[str]: return result +# Default context budget for prior files (metadata injection per file costs tokens +# in EVERY agent round). Override per workspace instance via instanceConfig +# key "maxPriorContextFiles". Files outside the budget remain fully accessible +# to the agent via conversation history and readFile - nothing is lost. +DEFAULT_MAX_PRIOR_CONTEXT_FILES = 10 + + +async def _selectPriorFilesForContext( + priorFileIds: List[str], + prompt: str, + aiObjects, + mandateId: str, + featureInstanceId: str, + maxFiles: int, +) -> List[str]: + """Select which prior files get their metadata injected into the agent context. + + Selection only happens when more candidates exist than the budget allows - + otherwise all files pass through untouched. Selection order: + 1. Files whose indexed chunks are semantically relevant to the current prompt + 2. Remaining budget filled with the most recent files (so files without an + index entry are never unfairly dropped) + """ + if len(priorFileIds) <= maxFiles: + return priorFileIds + + rankedFileIds: List[str] = [] + try: + embeddingResponse = await aiObjects.callEmbedding([prompt]) + embeddings = (embeddingResponse.metadata or {}).get("embeddings", []) + if embeddings: + knowledgeIf = getKnowledgeInterface() + results = knowledgeIf.semanticSearch( + queryVector=embeddings[0], + featureInstanceId=featureInstanceId, + mandateId=mandateId, + limit=maxFiles * 3, + ) + priorSet = set(priorFileIds) + for chunk in results: + fid = chunk.get("fileId") if isinstance(chunk, dict) else getattr(chunk, "fileId", None) + if fid and fid in priorSet and fid not in rankedFileIds: + rankedFileIds.append(fid) + if len(rankedFileIds) >= maxFiles: + break + except Exception as e: + logger.warning(f"Relevance ranking for prior files failed: {e}") + + for fid in reversed(priorFileIds): + if len(rankedFileIds) >= maxFiles: + break + if fid not in rankedFileIds: + rankedFileIds.append(fid) + + logger.info( + f"Prior-file selection: {len(priorFileIds)} candidates -> {len(rankedFileIds)} in context " + f"(budget={maxFiles}, relevance-ranked, recency fill)" + ) + return rankedFileIds + + +async def _ensureFilesIndexed( + fileIds: List[str], + user, + mandateId: str, + featureInstanceId: str, +) -> int: + """Ensure all attached files have embeddings in the knowledge store. + + Checks FileContentIndex for each file. Files not yet indexed are extracted + and embedded inline so that subsequent RAG queries can find their content. + Indexing is idempotent (content-hash check in requestIngestion), so each + file is only ever processed once - re-attaching an indexed file is a no-op. + + Returns the number of files that were newly indexed. + """ + if not fileIds: + return 0 + + knowledgeIf = getKnowledgeInterface(user) + unindexedIds = [] + for fid in fileIds: + existing = knowledgeIf.getFileContentIndex(fid) + status = (existing.get("status") if isinstance(existing, dict) else getattr(existing, "status", "")) if existing else "" + if status not in ("indexed", "embedding"): + unindexedIds.append(fid) + + if not unindexedIds: + return 0 + + logger.info(f"Ensure-embed: {len(unindexedIds)}/{len(fileIds)} files need indexing") + + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + indexCtx = ServiceCenterContext( + user=user, mandateId=mandateId, featureInstanceId=featureInstanceId, + ) + try: + knowledgeService = getService("knowledge", indexCtx) + except Exception as e: + logger.warning(f"Ensure-embed: knowledge service unavailable: {e}") + return 0 + + chatInterface = interfaceDbChat.getInterface(user) + indexed = 0 + for fid in unindexedIds: + try: + fileInfo = chatInterface.getFileInfo(fid) if chatInterface else None + if not fileInfo: + continue + fileName = fileInfo.get("fileName", "") + mimeType = fileInfo.get("mimeType", "") + rawBytes = chatInterface.getFileData(fid) + if not rawBytes: + continue + + extractionService = getService("extraction", indexCtx) + extracted = extractionService.extractContentFromBytes( + rawBytes, fileName, mimeType, documentId=fid, + ) + contentObjects = [ + { + "contentType": getattr(p, "contentType", "text"), + "data": getattr(p, "data", "") or "", + "contentObjectId": getattr(p, "contentObjectId", "") or str(uuid.uuid4()), + "contextRef": getattr(p, "contextRef", {}) or {}, + } + for p in (extracted.parts or []) + if getattr(p, "data", None) + ] + if not contentObjects: + continue + + await knowledgeService.indexFile( + fileId=fid, + fileName=fileName, + mimeType=mimeType, + userId=user.id if user else "", + featureInstanceId=featureInstanceId, + mandateId=mandateId, + contentObjects=contentObjects, + structure=getattr(extracted, "structure", None), + ) + indexed += 1 + except Exception as e: + logger.debug(f"Ensure-embed: skipping file {fid}: {e}") + + if indexed: + logger.info(f"Ensure-embed: indexed {indexed} file(s) before agent start") + return indexed + + async def _deriveWorkflowName(prompt: str, aiService) -> str: """Use AI to generate a concise workflow title from the user prompt.""" from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum @@ -740,20 +892,29 @@ async def _runWorkspaceAgent( priorFileIds = _collectPriorFileIds(chatInterface, workflowId) currentFileIdSet = set(fileIds or []) - mergedFileIds = list(fileIds or []) - for pf in priorFileIds: - if pf not in currentFileIdSet: - mergedFileIds.append(pf) - if len(mergedFileIds) > len(fileIds or []): + candidatePriorIds = [pf for pf in priorFileIds if pf not in currentFileIdSet] + + # Embed-first rule: newly attached files are indexed into the knowledge + # store BEFORE the agent starts, so RAG retrieval works from round 1. + await _ensureFilesIndexed(fileIds or [], user, mandateId, instanceId) + + _cfg = instanceConfig or {} + + if candidatePriorIds: + maxPriorFiles = int(_cfg.get("maxPriorContextFiles", DEFAULT_MAX_PRIOR_CONTEXT_FILES)) + candidatePriorIds = await _selectPriorFilesForContext( + candidatePriorIds, prompt, aiObjects, mandateId, instanceId, maxPriorFiles, + ) + + mergedFileIds = list(fileIds or []) + candidatePriorIds + if candidatePriorIds: logger.info( - f"Merged {len(mergedFileIds) - len(fileIds or [])} prior file(s) into agent context " + f"Merged {len(candidatePriorIds)} prior file(s) into agent context " f"(total: {len(mergedFileIds)}) for workflow {workflowId}" ) accumulatedText = "" messagePersisted = False - - _cfg = instanceConfig or {} _toolSet = _cfg.get("toolSet", "core") _agentCfg = _cfg.get("agentConfig") from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentConfig diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index c36e10b6..a1800648 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -4,7 +4,9 @@ import logging import asyncio import uuid import base64 +import hashlib import json +from collections import OrderedDict from typing import Dict, Any, List, Union, Tuple, Optional, Callable, AsyncGenerator from dataclasses import dataclass, field import time @@ -542,7 +544,27 @@ class AiObjects: else: options.operationType = OperationTypeEnum.EMBEDDING - combinedText = " ".join(texts[:3])[:500] + # Serve known vectors from cache; only unknown texts go to the API. + resolvedVectors: Dict[int, List[float]] = {} + pendingTexts: List[str] = [] + pendingPositions: List[int] = [] + for i, t in enumerate(texts): + cachedVector = _embeddingCacheGet(t) + if cachedVector is not None: + resolvedVectors[i] = cachedVector + else: + pendingTexts.append(t) + pendingPositions.append(i) + + if not pendingTexts: + logger.debug(f"Embedding cache hit for all {len(texts)} text(s)") + return AiCallResponse( + content="", modelName="embedding-cache", priceCHF=0.0, + processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=0, + metadata={"embeddings": [resolvedVectors[i] for i in range(len(texts))]}, + ) + + combinedText = " ".join(pendingTexts[:3])[:500] availableModels = modelRegistry.getAvailableModels() allowedProviders = getattr(options, 'allowedProviders', None) if options else None @@ -575,13 +597,13 @@ class AiObjects: for attempt, model in enumerate(failoverModelList): try: logger.info(f"Embedding call with {model.name} (attempt {attempt + 1}/{len(failoverModelList)})") - inputBytes = sum(len(t.encode("utf-8")) for t in texts) + inputBytes = sum(len(t.encode("utf-8")) for t in pendingTexts) startTime = time.time() - batches = _buildEmbeddingBatches(texts, model.contextLength) + batches = _buildEmbeddingBatches(pendingTexts, model.contextLength) logger.info( - f"Embedding: {len(texts)} texts -> {len(batches)} batch(es), " - f"model contextLength={model.contextLength}" + f"Embedding: {len(pendingTexts)} texts ({len(resolvedVectors)} cached) -> " + f"{len(batches)} batch(es), model contextLength={model.contextLength}" ) allEmbeddings: List[List[float]] = [] @@ -606,11 +628,17 @@ class AiObjects: if totalPriceCHF == 0.0: totalPriceCHF = model.calculatepriceCHF(processingTime, inputBytes, 0) + for j, position in enumerate(pendingPositions): + if j < len(allEmbeddings): + resolvedVectors[position] = allEmbeddings[j] + _embeddingCachePut(pendingTexts[j], allEmbeddings[j]) + mergedEmbeddings = [resolvedVectors.get(i, []) for i in range(len(texts))] + response = AiCallResponse( content="", modelName=model.name, provider=model.connectorType, priceCHF=totalPriceCHF, processingTime=processingTime, bytesSent=inputBytes, bytesReceived=0, errorCount=0, - metadata={"embeddings": allEmbeddings} + metadata={"embeddings": mergedEmbeddings} ) if self.billingCallback: @@ -681,6 +709,28 @@ class AiObjects: _CHARS_PER_TOKEN = 4 _SAFETY_MARGIN = 0.90 +# In-process cache for embedding vectors. Identical texts (e.g. the same user +# prompt embedded once for prior-file selection and once for RAG context +# building) hit the cache instead of paying for a second API call. +_EMBEDDING_CACHE_MAX_ENTRIES = 256 +_embeddingCache: OrderedDict = OrderedDict() + + +def _embeddingCacheGet(text: str) -> Optional[List[float]]: + key = hashlib.sha256(text.encode("utf-8")).hexdigest() + vector = _embeddingCache.get(key) + if vector is not None: + _embeddingCache.move_to_end(key) + return vector + + +def _embeddingCachePut(text: str, vector: List[float]) -> None: + key = hashlib.sha256(text.encode("utf-8")).hexdigest() + _embeddingCache[key] = vector + _embeddingCache.move_to_end(key) + while len(_embeddingCache) > _EMBEDDING_CACHE_MAX_ENTRIES: + _embeddingCache.popitem(last=False) + def _estimateTokens(text: str) -> int: """Rough token estimate: 1 token ~ 4 characters.""" @@ -691,9 +741,7 @@ def _buildEmbeddingBatches(texts: List[str], contextLength: int) -> List[List[st """Split a list of texts into batches whose total estimated token count stays within the model's contextLength (with safety margin). - Each individual text is assumed to already be within limits (enforced by - the chunking layer). If a single text exceeds the budget, it is placed - in its own batch as a last resort. + Texts that individually exceed the per-input limit are truncated to fit. """ if not texts: return [] @@ -701,11 +749,21 @@ def _buildEmbeddingBatches(texts: List[str], contextLength: int) -> List[List[st return [texts] maxTokensPerBatch = int(contextLength * _SAFETY_MARGIN) + maxCharsPerInput = maxTokensPerBatch * _CHARS_PER_TOKEN batches: List[List[str]] = [] currentBatch: List[str] = [] currentTokens = 0 for text in texts: + if len(text) > maxCharsPerInput: + # API hard limit per input. File content never hits this (the + # chunking layer splits at ~400 tokens); only oversized search + # queries can, where truncation is semantically acceptable. + logger.warning( + f"Embedding input truncated from {len(text)} to {maxCharsPerInput} chars " + f"(model input limit {contextLength} tokens)" + ) + text = text[:maxCharsPerInput] textTokens = _estimateTokens(text) if currentBatch and (currentTokens + textTokens) > maxTokensPerBatch: batches.append(currentBatch) diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 52cd5a59..a3dda3ba 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -3112,8 +3112,12 @@ class AppObjects: # Token methods - def saveAccessToken(self, token: Token, replace_existing: bool = True) -> None: - """Save an access token for the current user (must NOT have connectionId)""" + def saveAccessToken(self, token: Token, replace_existing: bool = False) -> None: + """Save an access token for the current user (must NOT have connectionId). + + Multi-session: replace_existing=False (default) keeps existing sessions alive. + Only set replace_existing=True for explicit single-session scenarios. + """ try: # Validate that this is NOT a connection token if token.connectionId: diff --git a/modules/routes/routeAdminSessions.py b/modules/routes/routeAdminSessions.py new file mode 100644 index 00000000..d962a9ea --- /dev/null +++ b/modules/routes/routeAdminSessions.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +""" +Admin endpoints for session and trusted device management. + +Allows mandate-admins and platform-admins to view and revoke active sessions +and trusted devices for users under their jurisdiction. +""" + +from fastapi import APIRouter, HTTPException, status, Depends, Request, Query +from typing import Dict, Any, List +import logging + +from modules.auth import limiter, getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelSecurity import Token, TokenPurpose, TokenStatus, TrustedDevice +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared.i18nRegistry import apiRouteContext + +routeApiMsg = apiRouteContext("routeAdminSessions") +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/admin/sessions", + tags=["Admin Sessions"], + responses={404: {"description": "Not found"}}, +) + + +def _requireAdmin(currentUser: User) -> None: + """Ensure the caller is a platform admin or sysAdmin.""" + if not (getattr(currentUser, "isPlatformAdmin", False) or getattr(currentUser, "isSysAdmin", False)): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("Only platform admins can manage sessions"), + ) + + +@router.get("") +@limiter.limit("30/minute") +def listSessions( + request: Request, + userId: str = Query(..., description="User ID whose sessions to list"), + currentUser: User = Depends(getCurrentUser), +) -> List[Dict[str, Any]]: + """List active auth sessions for a user.""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + tokens = rootInterface.db.getRecordset( + Token, + recordFilter={ + "userId": userId, + "tokenPurpose": TokenPurpose.AUTH_SESSION.value, + "status": TokenStatus.ACTIVE.value, + }, + ) + + now = getUtcTimestamp() + result = [] + for t in tokens: + expiresAt = t.get("expiresAt", 0) + if expiresAt < now: + continue + result.append({ + "sessionId": t.get("sessionId"), + "tokenId": t.get("id"), + "authority": t.get("authority"), + "createdAt": t.get("sysCreatedAt"), + "expiresAt": expiresAt, + }) + + return result + + +@router.delete("/{sessionId}") +@limiter.limit("30/minute") +def revokeSession( + request: Request, + sessionId: str, + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Revoke a single session by sessionId.""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + tokens = rootInterface.db.getRecordset( + Token, + recordFilter={"sessionId": sessionId, "tokenPurpose": TokenPurpose.AUTH_SESSION.value}, + ) + count = 0 + for t in tokens: + rootInterface.db.recordDelete(Token, t["id"]) + count += 1 + + if count == 0: + raise HTTPException(status_code=404, detail=routeApiMsg("Session not found")) + + logger.info(f"Admin {currentUser.username} revoked session {sessionId} ({count} token(s))") + return {"revoked": count, "sessionId": sessionId} + + +@router.delete("") +@limiter.limit("10/minute") +def revokeAllSessions( + request: Request, + userId: str = Query(..., description="User ID whose sessions to revoke"), + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Revoke ALL active sessions for a user (force logout everywhere).""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + tokens = rootInterface.db.getRecordset( + Token, + recordFilter={ + "userId": userId, + "tokenPurpose": TokenPurpose.AUTH_SESSION.value, + }, + ) + count = 0 + for t in tokens: + rootInterface.db.recordDelete(Token, t["id"]) + count += 1 + + logger.info(f"Admin {currentUser.username} revoked all sessions for userId={userId} ({count} token(s))") + return {"revoked": count, "userId": userId} + + +# --- Trusted Devices --- + +trustedDeviceRouter = APIRouter( + prefix="/api/admin/trusted-devices", + tags=["Admin Sessions"], + responses={404: {"description": "Not found"}}, +) + + +@trustedDeviceRouter.get("") +@limiter.limit("30/minute") +def listTrustedDevices( + request: Request, + userId: str = Query(..., description="User ID whose trusted devices to list"), + currentUser: User = Depends(getCurrentUser), +) -> List[Dict[str, Any]]: + """List trusted devices for a user.""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + devices = rootInterface.db.getRecordset( + TrustedDevice, recordFilter={"userId": userId} + ) + + now = getUtcTimestamp() + result = [] + for d in devices: + result.append({ + "id": d.get("id", "")[:8] + "...", + "trustedUntil": d.get("trustedUntil"), + "isExpired": d.get("trustedUntil", 0) < now, + "userAgent": d.get("userAgent"), + "ipAddress": d.get("ipAddress"), + "createdAt": d.get("createdAt"), + }) + + return result + + +@trustedDeviceRouter.delete("") +@limiter.limit("10/minute") +def revokeAllTrustedDevices( + request: Request, + userId: str = Query(..., description="User ID whose trusted devices to revoke"), + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Revoke ALL trusted devices for a user (force MFA on next login).""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + from modules.auth.trustedDeviceService import revokeTrustedDevices + count = revokeTrustedDevices(userId, rootInterface.db) + + logger.info(f"Admin {currentUser.username} revoked all trusted devices for userId={userId} ({count})") + return {"revoked": count, "userId": userId} diff --git a/modules/routes/routeMfa.py b/modules/routes/routeMfa.py index 0d3e4d59..5b13d592 100644 --- a/modules/routes/routeMfa.py +++ b/modules/routes/routeMfa.py @@ -232,6 +232,13 @@ def mfaVerify( logger.info("MFA verify successful for user %s", username) + # Mark device as trusted so MFA is skipped on next login from this device + try: + from modules.auth.trustedDeviceService import createTrustedDevice + createTrustedDevice(userId, request, response, rootInterface.db) + except Exception as e: + logger.warning(f"Failed to create trusted device after MFA verify: {e}") + try: from modules.dbHelpers.auditLogger import audit_logger audit_logger.logUserAccess( diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index ee2f6390..3107a2b8 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -256,6 +256,7 @@ def login( # --- MFA gate -------------------------------------------------------- from modules.auth.mfaService import isMfaRequired as _isMfaRequired + from modules.auth.trustedDeviceService import isTrustedDevice as _isTrustedDevice from modules.routes.routeMfa import createMfaPendingToken userRecord = rootInterface._getUserForAuthentication(user.username) @@ -273,7 +274,14 @@ def login( mfaRequired = _isMfaRequired(user, userMandates=userMandates, mandates=mandateObjs) hasMfaSetup = bool(userRecord and userRecord.get("mfaSecret") and getattr(user, "mfaEnabled", False)) + # Trusted device: skip MFA if the device was previously verified + _deviceTrusted = False if mfaRequired or hasMfaSetup: + _deviceTrusted = _isTrustedDevice(request, str(user.id), rootInterface.db) + if _deviceTrusted: + logger.info(f"MFA skipped for user {user.username} (trusted device)") + + if (mfaRequired or hasMfaSetup) and not _deviceTrusted: _sid = str(uuid.uuid4()) pendingToken = createMfaPendingToken( userId=str(user.id), @@ -659,31 +667,46 @@ def refresh_token( logger.error(f"Failed to get user from database: {str(e)}") raise HTTPException(status_code=500, detail=routeApiMsg("Failed to validate user")) + # Preserve sessionId from the refresh token so the session stays grouped + sessionId = payload.get("sid") or str(uuid.uuid4()) + # Create new token data - # MULTI-TENANT: Token does NOT contain mandateId anymore + newJti = str(uuid.uuid4()) token_data = { "sub": current_user.username, "userId": str(current_user.id), - "authenticationAuthority": current_user.authenticationAuthority - # NO mandateId in token + "authenticationAuthority": current_user.authenticationAuthority, + "jti": newJti, + "sid": sessionId, } - + # Create new access token + set cookie - access_token, _expires = createAccessToken(token_data) + access_token, accessExpires = createAccessToken(token_data) setAccessTokenCookie(response, access_token) - - # Get expiration time + + # Persist the new token in DB so _getUserBase() accepts it + authority = current_user.authenticationAuthority + if isinstance(authority, str): + authority = AuthAuthority(authority) + dbToken = Token( + id=newJti, + userId=str(current_user.id), + authority=authority, + tokenAccess=access_token, + tokenPurpose=TokenPurpose.AUTH_SESSION, + expiresAt=accessExpires.timestamp(), + sessionId=sessionId, + ) try: - payload = jwt.decode(access_token, SECRET_KEY, algorithms=[ALGORITHM]) - expires_at = datetime.fromtimestamp(payload.get("exp")) + userInterface = getInterface(current_user) + userInterface.saveAccessToken(dbToken) except Exception as e: - logger.error(f"Failed to decode new access token: {str(e)}") - raise HTTPException(status_code=500, detail=routeApiMsg("Failed to create new token")) - + logger.warning(f"Failed to persist refreshed token in DB: {e}") + return { "type": "token_refresh_success", "message": "Token refreshed successfully", - "expires_at": expires_at.isoformat() + "expires_at": accessExpires.isoformat() } except HTTPException as e: diff --git a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py index d2c0830b..6ad29488 100644 --- a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py +++ b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py @@ -638,7 +638,8 @@ class KnowledgeService: Returns: Formatted context string for injection into the agent's system prompt. """ - queryVector = await self._embedSingle(currentPrompt) + queryText = _extractUserQuery(currentPrompt) + queryVector = await self._embedSingle(queryText) if queryText else [] logger.debug( "buildAgentContext.start userId=%s featureInstanceId=%s mandateId=%s isSysAdmin=%s prompt=%r", userId, featureInstanceId, mandateId, isSysAdmin, (currentPrompt or "")[:120], @@ -960,6 +961,29 @@ class KnowledgeService: # Internal helpers # ============================================================================= +# Markers added by the prompt enrichment layers (_enrichPromptWithFiles in +# mainServiceAgent, data source sections in routeFeatureWorkspace). Used to +# isolate the user's own words for the semantic search query. +_USER_REQUEST_MARKER = "\n\nUser request: " +_PROMPT_SECTION_MARKERS = ("\n\n[Active Data Sources]\n", "\n\n[Attached Feature Data Sources]\n") + + +def _extractUserQuery(prompt: str) -> str: + """Isolate the user's actual question from an enriched agent prompt. + + Enriched prompts wrap the user request in file metadata headers and + data-source sections. Only the user's own words form a meaningful semantic + search query - embedding the metadata would dilute the vector and can + exceed the embedding model's input limit. + """ + if not prompt: + return "" + text = prompt.rsplit(_USER_REQUEST_MARKER, 1)[-1] + for marker in _PROMPT_SECTION_MARKERS: + text = text.split(marker, 1)[0] + return text.strip() + + def _estimateTokens(text: str) -> int: """Estimate token count using character-based heuristic (1 token ~ 4 chars).""" return max(1, len(text) // CHARS_PER_TOKEN) From 63e30c128182ddaaa829651a37961855fd40da67 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 11 Jun 2026 22:54:39 +0200 Subject: [PATCH 4/8] import cleanup --- app.py | 10 +- env-dev.env | 2 +- env-int.env | 2 +- env-prod.env | 2 +- modules/auth/trustedDeviceService.py | 48 + .../commcoach/interfaceFeatureCommcoach.py | 11 - .../commcoach/routeFeatureCommcoach.py | 5 - .../features/commcoach/serviceCommcoach.py | 4 - .../serviceNeutralization/subProcessList.py | 2 - .../realEstate/interfaceFeatureRealEstate.py | 1 - .../features/teamsbot/routeFeatureTeamsbot.py | 9 - .../trustee/accounting/accountingBridge.py | 4 - .../trustee/interfaceFeatureTrustee.py | 2 - .../features/trustee/routeFeatureTrustee.py | 7 - .../methodTrustee/actions/extractFromFiles.py | 1 - .../workspace/routeFeatureWorkspace.py | 1 - modules/interfaces/interfaceAiObjects.py | 2 - modules/interfaces/interfaceDbApp.py | 18 - modules/interfaces/interfaceDbBilling.py | 2 - modules/interfaces/interfaceDbChat.py | 2 - modules/interfaces/interfaceDbKnowledge.py | 1 - modules/interfaces/interfaceDbManagement.py | 5 - modules/interfaces/interfaceRbac.py | 6 - modules/interfaces/interfaceTableHelpers.py | 2 - modules/routes/routeAdminFeatures.py | 1 - modules/routes/routeAdminRbacRules.py | 3 - modules/routes/routeBilling.py | 2 - modules/routes/routeDataConnections.py | 3 - modules/routes/routeDataFiles.py | 2 - modules/routes/routeDataPrompts.py | 1 - modules/routes/routeMfa.py | 1 - modules/routes/routeNotifications.py | 1 - modules/routes/routeRagInventory.py | 1 - modules/routes/routeSecurityLocal.py | 4 - modules/routes/routeWorkflowAutomation.py | 7 - .../services/serviceAgent/mainServiceAgent.py | 7 - .../services/serviceAi/mainServiceAi.py | 2 - .../services/serviceChat/mainServiceChat.py | 2 - .../extractors/extractorContainer.py | 1 - .../extractors/extractorEmail.py | 1 - .../extractors/extractorFolder.py | 1 - .../services/serviceExtraction/subRegistry.py | 2 - .../renderers/rendererPdf.py | 1 - .../renderers/rendererPptx.py | 1 - .../renderers/rendererXlsx.py | 1 - .../serviceKnowledge/subConnectorSyncGmail.py | 5 +- .../mainServiceSubscription.py | 2 - modules/system/databaseHealth.py | 2 - .../engine/executors/actionNodeExecutor.py | 1 - modules/workflowAutomation/helpers.py | 2 - .../mainWorkflowAutomation.py | 1 - .../methods/methodAi/actions/process.py | 2 - .../script_analyze_platform_module_graph.py | 568 +++++++++++ scripts/script_analyze_porta_imports.py | 898 ++++++++++++++++++ ...cript_remove_redundant_platform_imports.py | 165 ++++ 55 files changed, 1691 insertions(+), 149 deletions(-) create mode 100644 scripts/script_analyze_platform_module_graph.py create mode 100644 scripts/script_analyze_porta_imports.py create mode 100644 scripts/script_remove_redundant_platform_imports.py diff --git a/app.py b/app.py index 1ce31ae5..1df8510f 100644 --- a/app.py +++ b/app.py @@ -380,7 +380,6 @@ async def lifespan(app: FastAPI): # Register all feature definitions in RBAC catalog (for /api/features/ endpoint) try: from modules.security.rbacCatalog import getCatalogService - from modules.system.registry import registerAllFeaturesInCatalog, syncCatalogFeaturesToDb catalogService = getCatalogService() registerAllFeaturesInCatalog(catalogService) logger.info("Feature catalog registration completed") @@ -494,7 +493,6 @@ async def lifespan(app: FastAPI): def _onRunFailed(workflowId, runId, error, mandateId=None, workflowLabel=None): from modules.serviceCenter import getService from modules.serviceCenter.context import ServiceCenterContext - from modules.interfaces.interfaceDbApp import getRootInterface from modules.datamodels.datamodelMessaging import MessagingEventParameters rootInterface = getRootInterface() @@ -555,6 +553,10 @@ async def lifespan(app: FastAPI): from modules.serviceCenter.services.serviceSubscription.enterpriseRenewalScheduler import registerEnterpriseRenewalScheduler registerEnterpriseRenewalScheduler() + # Register token and trusted device cleanup scheduler + from modules.auth.trustedDeviceService import registerTokenCleanupScheduler + registerTokenCleanupScheduler() + # Recover background jobs that were RUNNING when the previous worker died try: from modules.serviceCenter.services.serviceBackgroundJobs.mainBackgroundJobService import ( @@ -901,6 +903,10 @@ app.include_router(demoConfigRouter) from modules.routes.routeAdminDatabaseHealth import router as adminDatabaseHealthRouter app.include_router(adminDatabaseHealthRouter) +from modules.routes.routeAdminSessions import router as adminSessionsRouter, trustedDeviceRouter as adminTrustedDeviceRouter +app.include_router(adminSessionsRouter) +app.include_router(adminTrustedDeviceRouter) + from modules.routes.routeGdpr import router as gdprRouter app.include_router(gdprRouter) diff --git a/env-dev.env b/env-dev.env index 457cc7a5..179f7caf 100644 --- a/env-dev.env +++ b/env-dev.env @@ -58,7 +58,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = http://localhost:8000/api/clickup/auth/conn # Stripe Billing (both end with _SECRET for encryption script) STRIPE_SECRET_KEY_SECRET = DEV_ENC:Z0FBQUFBQnB5dkd5aHNGejgzQmpTdmprdzQxR19KZkh3MlhYUTNseFN3WnlaWjh2SDZyalN6aU9xSktkbUQwUnZrVnlvbGVRQm4yZFdiRU5aSEk5WVJuUnR4VUwtTm9OVk1WWmJQeU5QaDdib0hfVWV5U1BfYTFXRmdoOWdnOWxkb3JFQmF3bm45UjFUVUxmWGtGRkFKUGd6bmhpQlFnaVI3Q2lLdDlsY1VESk1vOEM0ZFBJNW1qcVZ0N2tPYmRLNmVKajZ2M3o3S05lWnRRVG5LdkRseW4wQ3VjNHNQZTZUdz09 -STRIPE_WEBHOOK_SECRET = DEV_ENC:Z0FBQUFBQnB5dkd5dDJMSHBrVk8wTzJhU2xzTTZCZWdvWmU2NGI2WklfRXRJZVUzaVYyOU9GLUZsalUwa2lPdEgtUHo0dVVvRDU1cy1saHJyU0Rxa2xQZjBuakExQzk3bmxBcU9WbEIxUEtpR1JoUFMxZG9ISGRZUXFhdFpSMGxvQUV3a0VLQllfUUtCOHZwTGdteV9rYTFOazBfSlN3ekNWblFpakJlZVlCTmNkWWQ4Sm01a1RCWTlnTlFHWVA0MkZYMlprUExrWFN2V0NVU1BTd1NKczFJbVo3VHpLdlc4UT09 +STRIPE_WEBHOOK_SECRET = DEV_ENC:Z0FBQUFBQnFLeUFlb2dfSjZPaWIyRjZsNjhiSDFQNFpxdW50YmlLUjFLX1lJMGdCWUtBUEdrRGhvSzVVWnkxNVZEdmtkQmk5X05YS0JVU1NyX3VQZTV2VjVwakd0RGM2WUl6TTlzbms1d1NCOTQtdURiVjhxdXZGVlR1ZVNTbUkwOFh1R04yUUxxay0= STRIPE_API_VERSION = 2026-01-28.clover STRIPE_AUTOMATIC_TAX_ENABLED = false STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQd14OUoIL0Osj7A0ZQlr0 diff --git a/env-int.env b/env-int.env index 84e0feb4..a8f67e6f 100644 --- a/env-int.env +++ b/env-int.env @@ -60,7 +60,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = https://api-int.poweron.swiss/api/clickup/a # Stripe Billing (both end with _SECRET for encryption script) STRIPE_SECRET_KEY_SECRET = INT_ENC:Z0FBQUFBQnB5dkd5ekdBaGNGVUlOQUpncTlzLWlTV0V5OWZzQkpDczhCUGw4U1JpTHZ0d3pfYlFNWElLRlNiNlNsaDRYTGZUTkg2OUFrTW1GZXpOUjBVbmRQWjN6ekhHd2ZSQ195OHlaeWh1TmxrUm10V2R3YmdncmFLbFMzVjdqcWJMSUJPR2xuSEozclNoZG1rZVBTaWg3OFQ1Qzdxb0wyQ2RKazc2dG1aZXBUTXlvbDZqLS1KOVI5M3BGc3NQZkZRbnFpRjIwWmh2ZHlVNlpxZVo2dWNmMjQ5eW02QmtzUT09 -STRIPE_WEBHOOK_SECRET = INT_ENC:Z0FBQUFBQnA4UXZiUUVqTl9lREVRWTh1aHFDcFpwcXRkOUx4MS1ham9Ddkl6T0xzMnJuM1hhUHdGNG5CenY1MUg4RlJBOGFQTWl5cVd5MjJ2REItcHYyRmdLX3ZlT2p5Z3BRVkMtQnRoTVkteXlfaU92MVBtOEI0Ni1kbGlfa0NiRmFRRXNHLVE2NHI= +STRIPE_WEBHOOK_SECRET = INT_ENC:Z0FBQUFBQnFLeUFWUUtMZ25NQ2ZOWE5nRF9CaFNwcXhSU2tKRktLaElLRHJMM295OXNkVEFLekVUMzN0YUpIZHJfWGNqa0xxOFZRVHZEUXVLZ3ItVGZWc2VFQ2thcUlJalY1b0JDSmR6RF96d1A3OGhyd0w1MHZPeFNZRkl0c19kYUJQcHVwR2tsd0s= STRIPE_API_VERSION = 2026-01-28.clover STRIPE_AUTOMATIC_TAX_ENABLED = false STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQd14OUoIL0Osj7A0ZQlr0 diff --git a/env-prod.env b/env-prod.env index 6a2a89d7..686f784b 100644 --- a/env-prod.env +++ b/env-prod.env @@ -58,7 +58,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = https://api.poweron.swiss/api/clickup/auth/ # Stripe Billing (both end with _SECRET for encryption script) STRIPE_SECRET_KEY_SECRET = PROD_ENC:Z0FBQUFBQnB5dkd6aVA3R3VRS3VHMUgzUEVjYkR4eUZKWFhPUzFTTVlHNnBvT3FienNQaUlBWVpPLXJyVGpGMWk4LXktMXphX0J6ZTVESkJxdjNNa3ZJbF9wX2ppYzdjYlF0cmdVamlEWWJDSmJYYkJseHctTlh4dnNoQWs4SG5haVl2TTNDdXpuaFpqeDBtNkFCbUxMa0RaWG14dmxyOEdILTNrZ2licmNpbXVkN2lFSWoxZW1BODNpV0ZTQ0VaeXRmR1d4RjExMlVFS3MtQU9zZXZlZE1mTmY3OWctUXJHdz09 -STRIPE_WEBHOOK_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGNUpTWldsakYydFhFelBrR1lSaWxYT3kyMENOMUljZTJUZHBWcEhhdWVCMzYxZXQ5b3VlTFVRalFiTVdsbGxrdUx0RDFwSEpsOC1sTDJRTEJNQlA3S3ZaQzBtV1h6bWp5VnlMZUgwUlF3cXYxcnljZVE5SWdzLVg3V0syOWRYS08= +STRIPE_WEBHOOK_SECRET = PROD_ENC:Z0FBQUFBQnFLeUFNQ1FhVE94ZzM3V3NCVGVVWnltUndsOG1Ra0hQTmJ3QWY5aXVWeTJsX3A4a3VBSnFQd3drWFRZNFVDdWxCeFgyQ0RpNGQ0SlJOcm9tVE5KZmVqQU1WUjFjeDRJeGE5THdmR0g1V2dQUk5SSjcySnAzR245NW5NUFVDT3lJUWpjWFo= STRIPE_API_VERSION = 2026-01-28.clover STRIPE_AUTOMATIC_TAX_ENABLED = false STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQZG8WqlVsabrfFEu49pah diff --git a/modules/auth/trustedDeviceService.py b/modules/auth/trustedDeviceService.py index a707e70a..d57d5c63 100644 --- a/modules/auth/trustedDeviceService.py +++ b/modules/auth/trustedDeviceService.py @@ -169,3 +169,51 @@ def _getClientIp(request: Request) -> Optional[str]: if request.client: return request.client.host return None + + +# --- Scheduler Integration --- + +async def _runTokenAndDeviceCleanup() -> None: + """Scheduled task: remove expired tokens and trusted devices.""" + try: + from modules.connectors.connectorDbPostgre import ConnectorPostgre + + db = ConnectorPostgre("poweron_app") + now = getUtcTimestamp() + + # Expired auth-session tokens + tokens = db.getRecordset( + Token, + recordFilter={"tokenPurpose": TokenPurpose.AUTH_SESSION.value}, + ) + expiredCount = 0 + for t in tokens: + if t.get("expiresAt", 0) < now: + db.recordDelete(Token, t["id"]) + expiredCount += 1 + + # Expired trusted devices + deviceCount = cleanupExpiredDevices(db) + + if expiredCount or deviceCount: + logger.info( + f"Token cleanup: {expiredCount} expired token(s), " + f"{deviceCount} expired trusted device(s) removed" + ) + except Exception as e: + logger.error(f"Token/device cleanup failed: {e}") + + +def registerTokenCleanupScheduler() -> None: + """Register daily token cleanup job. Call during app startup.""" + try: + from modules.shared.eventManagement import eventManager + + eventManager.registerCron( + jobId="token_device_cleanup", + func=_runTokenAndDeviceCleanup, + cronKwargs={"hour": "4", "minute": "0"}, + ) + logger.info("Token/device cleanup scheduler registered (daily 04:00)") + except Exception as e: + logger.warning(f"Failed to register token cleanup scheduler: {e}") diff --git a/modules/features/commcoach/interfaceFeatureCommcoach.py b/modules/features/commcoach/interfaceFeatureCommcoach.py index d4c51a27..a2e18770 100644 --- a/modules/features/commcoach/interfaceFeatureCommcoach.py +++ b/modules/features/commcoach/interfaceFeatureCommcoach.py @@ -261,35 +261,29 @@ class CommcoachObjects: # ========================================================================= def getPersonas(self, userId: str, instanceId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import CoachingPersona builtins = self.db.getRecordset(CoachingPersona, recordFilter={"userId": "system"}) custom = self.db.getRecordset(CoachingPersona, recordFilter={"userId": userId, "instanceId": instanceId}) all = builtins + custom return [p for p in all if p.get("isActive", True)] def getPersona(self, personaId: str) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingPersona records = self.db.getRecordset(CoachingPersona, recordFilter={"id": personaId}) return records[0] if records else None def createPersona(self, data: Dict[str, Any]) -> Dict[str, Any]: - from .datamodelCommcoach import CoachingPersona data["createdAt"] = getIsoTimestamp() data["updatedAt"] = getIsoTimestamp() return self.db.recordCreate(CoachingPersona, data) def updatePersona(self, personaId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingPersona updates["updatedAt"] = getIsoTimestamp() return self.db.recordModify(CoachingPersona, personaId, updates) def deletePersona(self, personaId: str) -> bool: - from .datamodelCommcoach import CoachingPersona return self.db.recordDelete(CoachingPersona, personaId) def getAllPersonas(self, instanceId: str) -> List[Dict[str, Any]]: """All personas (builtin + custom for this instance), including inactive.""" - from .datamodelCommcoach import CoachingPersona builtins = self.db.getRecordset(CoachingPersona, recordFilter={"userId": "system"}) custom = self.db.getRecordset(CoachingPersona, recordFilter={"instanceId": instanceId}) custom = [p for p in custom if p.get("userId") != "system"] @@ -300,11 +294,9 @@ class CommcoachObjects: # ========================================================================= def getModulePersonas(self, moduleId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import ModulePersonaMapping return self.db.getRecordset(ModulePersonaMapping, recordFilter={"moduleId": moduleId}) def setModulePersonas(self, moduleId: str, personaIds: List[str], instanceId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import ModulePersonaMapping existing = self.db.getRecordset(ModulePersonaMapping, recordFilter={"moduleId": moduleId}) for rec in existing: self.db.recordDelete(ModulePersonaMapping, rec["id"]) @@ -325,18 +317,15 @@ class CommcoachObjects: # ========================================================================= def getBadges(self, userId: str, instanceId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import CoachingBadge records = self.db.getRecordset(CoachingBadge, recordFilter={"userId": userId, "instanceId": instanceId}) records.sort(key=lambda r: r.get("awardedAt") or 0, reverse=True) return records def hasBadge(self, userId: str, instanceId: str, badgeKey: str) -> bool: - from .datamodelCommcoach import CoachingBadge records = self.db.getRecordset(CoachingBadge, recordFilter={"userId": userId, "instanceId": instanceId, "badgeKey": badgeKey}) return len(records) > 0 def awardBadge(self, data: Dict[str, Any]) -> Dict[str, Any]: - from .datamodelCommcoach import CoachingBadge data["awardedAt"] = getUtcTimestamp() data["createdAt"] = getIsoTimestamp() return self.db.recordCreate(CoachingBadge, data) diff --git a/modules/features/commcoach/routeFeatureCommcoach.py b/modules/features/commcoach/routeFeatureCommcoach.py index 81e1254d..905ffbc9 100644 --- a/modules/features/commcoach/routeFeatureCommcoach.py +++ b/modules/features/commcoach/routeFeatureCommcoach.py @@ -333,7 +333,6 @@ async def startSession( try: from modules.interfaces.interfaceVoiceObjects import getVoiceInterface voiceInterface = getVoiceInterface(context.user, mandateId) - from .serviceCommcoach import getUserVoicePrefs, stripMarkdownForTts, buildTtsConfigErrorMessage language, voiceName = getUserVoicePrefs(userId, mandateId) ttsResult = await voiceInterface.textToSpeech( text=stripMarkdownForTts(greetingText), @@ -378,7 +377,6 @@ async def startSession( asyncio.create_task(service.processSessionOpening(sessionId, moduleId, interface)) async def _newSessionEventGenerator(): - from modules.shared.timeUtils import getIsoTimestamp timeoutCount = 0 try: while True: @@ -468,7 +466,6 @@ async def cancelSession( raise HTTPException(status_code=404, detail=routeApiMsg("Session not found")) _validateOwnership(session, context) - from modules.shared.timeUtils import getUtcTimestamp interface.updateSession(sessionId, { "status": CoachingSessionStatus.CANCELLED.value, "endedAt": getUtcTimestamp(), @@ -581,7 +578,6 @@ async def sendAudioStream( if not audioBody: raise HTTPException(status_code=400, detail=routeApiMsg("No audio data received")) - from .serviceCommcoach import getUserVoicePrefs language, _ = getUserVoicePrefs(str(context.user.id), mandateId) moduleId = session.get("moduleId") @@ -765,7 +761,6 @@ async def updateTaskStatus( updates = {"status": body.status.value} if body.status == CoachingTaskStatus.DONE: - from modules.shared.timeUtils import getUtcTimestamp updates["completedAt"] = getUtcTimestamp() updated = interface.updateTask(taskId, updates) diff --git a/modules/features/commcoach/serviceCommcoach.py b/modules/features/commcoach/serviceCommcoach.py index b3b5ef2a..315a9dff 100644 --- a/modules/features/commcoach/serviceCommcoach.py +++ b/modules/features/commcoach/serviceCommcoach.py @@ -98,7 +98,6 @@ def getUserVoicePrefs(userId: str, mandateId: Optional[str] = None) -> tuple: """Load voice language and voiceName from central UserVoicePreferences. Returns (language, voiceName) tuple.""" try: - from modules.datamodels.datamodelUam import UserVoicePreferences from modules.interfaces.interfaceDbApp import getRootInterface rootIf = getRootInterface() prefs = rootIf.db.getRecordset( @@ -430,7 +429,6 @@ async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, A """Pre-AI-call: identify which documents the user references and what action is needed.""" if not docs: return {"read": [], "update": [], "create": [], "noDocumentAction": True} - from . import serviceCommcoachAi as aiPrompts docCatalog = [{"id": d.get("id", ""), "title": d.get("summary") or d.get("fileName", ""), "summary": (d.get("summary") or "")[:100]} for d in docs] prompt = aiPrompts.buildDocumentIntentPrompt(combinedUserPrompt, docCatalog) try: @@ -744,7 +742,6 @@ class CommcoachService: 4. Map agent events to CommCoach SSE events 5. Post-processing: store message, TTS, tasks, scores """ - from . import interfaceFeatureCommcoach as interfaceDb # Store user message userMsg = CoachingMessage( @@ -907,7 +904,6 @@ class CommcoachService: ) agentService = getService("agent", serviceContext) - from modules.datamodels.datamodelAi import PriorityEnum, OperationTypeEnum config = AgentConfig( toolSet="commcoach" if useTools else "none", maxRounds=3 if useTools else 1, diff --git a/modules/features/neutralization/serviceNeutralization/subProcessList.py b/modules/features/neutralization/serviceNeutralization/subProcessList.py index a42904ff..ecc14b78 100644 --- a/modules/features/neutralization/serviceNeutralization/subProcessList.py +++ b/modules/features/neutralization/serviceNeutralization/subProcessList.py @@ -157,7 +157,6 @@ class ListProcessor: processedAttrs[attrName] = self.string_parser.mapping[attrValue] else: # Check if attribute value matches any data patterns - from .subPatterns import findPatternsInText, DataPatterns matches = findPatternsInText(attrValue, DataPatterns.patterns) if matches: patternName = matches[0][0] @@ -191,7 +190,6 @@ class ListProcessor: # Skip if already a placeholder if not self.string_parser._isPlaceholder(text): # Check if text matches any patterns - from .subPatterns import findPatternsInText, DataPatterns patternMatches = findPatternsInText(text, DataPatterns.patterns) if patternMatches: diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 24fe4955..5d1ea3a6 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -796,7 +796,6 @@ class RealEstateObjects: return False tableName = modelClass.__name__ - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None) permissions = self.rbac.getUserPermissions( self.currentUser, diff --git a/modules/features/teamsbot/routeFeatureTeamsbot.py b/modules/features/teamsbot/routeFeatureTeamsbot.py index b2ac2980..c4219ac7 100644 --- a/modules/features/teamsbot/routeFeatureTeamsbot.py +++ b/modules/features/teamsbot/routeFeatureTeamsbot.py @@ -328,7 +328,6 @@ async def startSession( if context.isSysAdmin and joinMode == TeamsbotJoinMode.SYSTEM_BOT: systemBot = interface.getActiveSystemBot(mandateId) if not systemBot: - from .datamodelTeamsbot import TeamsbotSystemBot allBots = interface.db.getRecordset(TeamsbotSystemBot, recordFilter={"isActive": True}) if allBots: systemBot = allBots[0] @@ -537,7 +536,6 @@ async def streamSession( async def _eventGenerator(): """Generate SSE events from the session event queue.""" - from .service import sessionEvents # Send initial session state with stats stats = interface.getSessionStats(sessionId) @@ -545,7 +543,6 @@ async def streamSession( # Send current bot WebSocket connection state so the operator UI can # render the live indicator without waiting for the next connect/disconnect. - from .service import getActiveService as _getActiveService yield f"data: {json.dumps({'type': 'botConnectionState', 'data': {'connected': _getActiveService(sessionId) is not None}})}\n\n" # Stream events @@ -1040,7 +1037,6 @@ async def submitDirectorPrompt( detail=routeApiMsg(f"Too many files ({len(fileIds)}); max {DIRECTOR_PROMPT_FILE_LIMIT}"), ) - from .service import getActiveService service = getActiveService(sessionId) if not service: raise HTTPException( @@ -1108,7 +1104,6 @@ async def deleteDirectorPrompt( if not context.isPlatformAdmin and prompt.get("operatorUserId") != str(context.user.id): raise HTTPException(status_code=404, detail=f"Prompt '{promptId}' not found") - from .service import getActiveService service = getActiveService(sessionId) if service: await service.removePersistentPrompt(promptId) @@ -1134,7 +1129,6 @@ async def testVoice( ): """Test TTS voice with AI-generated sample text in the correct language.""" from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - from .service import createAiService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum mandateId = _validateInstanceAccess(instanceId, context) @@ -1547,7 +1541,6 @@ async def postTranscript( originalUser = rootUser # Process transcript through the service pipeline - from .service import TeamsbotService from modules.interfaces.interfaceVoiceObjects import getVoiceInterface service = TeamsbotService(originalUser, mandateId, instanceId, config) @@ -1600,7 +1593,6 @@ async def postBotStatus( if not originalUser: originalUser = rootUser - from .service import TeamsbotService service = TeamsbotService(originalUser, mandateId, instanceId, config) interface = interfaceDb.getInterface(originalUser, mandateId=mandateId, featureInstanceId=instanceId) @@ -1640,7 +1632,6 @@ async def botWebsocket( # Load the original user who started the session (has RBAC roles in mandate) # Bot callbacks have no HTTP auth, so we reconstruct the user context from the session record. - from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() rootUser = rootInterface.currentUser diff --git a/modules/features/trustee/accounting/accountingBridge.py b/modules/features/trustee/accounting/accountingBridge.py index 51433bbf..d9b9fe73 100644 --- a/modules/features/trustee/accounting/accountingBridge.py +++ b/modules/features/trustee/accounting/accountingBridge.py @@ -33,7 +33,6 @@ class AccountingBridge: async def getActiveConfig(self, featureInstanceId: str) -> Optional[Dict[str, Any]]: """Load the active TrusteeAccountingConfig for a feature instance.""" - from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig records = self._trusteeInterface.db.getRecordset( TrusteeAccountingConfig, recordFilter={"featureInstanceId": featureInstanceId, "isActive": True}, @@ -128,7 +127,6 @@ class AccountingBridge: Optional _resolved* params allow pushBatchToAccounting to pass a pre-resolved connector/config so we don't decrypt per position (avoids rate-limit). """ - from modules.features.trustee.datamodelFeatureTrustee import TrusteePosition, TrusteeAccountingSync connector = _resolvedConnector plainConfig = _resolvedPlainConfig @@ -306,7 +304,6 @@ class AccountingBridge: # Update last sync on config record if configRecord: - from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig updatePayload = { "lastSyncAt": time.time(), "lastSyncStatus": "success" if result.success else "error", @@ -335,7 +332,6 @@ class AccountingBridge: async def refreshChartOfAccounts(self, featureInstanceId: str) -> List[AccountingChart]: """Fetch the full chart of accounts from the external system and cache it locally on TrusteeAccountingConfig.""" - from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig connector, plainConfig, configRecord = await self._resolveConnectorAndConfig(featureInstanceId) if not connector or not plainConfig or not configRecord: diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index 1e13f185..249b2d50 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -309,7 +309,6 @@ class TrusteeObjects: return False tableName = modelClass.__name__ - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None) permissions = self.rbac.getUserPermissions( self.currentUser, @@ -338,7 +337,6 @@ class TrusteeObjects: return AccessLevel.NONE tableName = modelClass.__name__ - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None) permissions = self.rbac.getUserPermissions( self.currentUser, diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index dbc96013..6e6b9924 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -170,7 +170,6 @@ def getQuickActions( if role and role.roleLabel: userRoleLabels.add(role.roleLabel) - from modules.shared.i18nRegistry import resolveText lang = (language or "de").strip() or "de" @@ -1201,7 +1200,6 @@ def _buildSyncStatusByPosition(interface, instanceId: str) -> Dict[str, Dict[str ``error``, so a successful retry hides an old failure. Any other status (`pending`, `cancelled`, ...) is kept verbatim. """ - from .datamodelFeatureTrustee import TrusteeAccountingSync syncRecords = interface.db.getRecordset( TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId} @@ -1290,7 +1288,6 @@ def _handlePositionMode(instanceId, mandateId, mode, column, pagination, context """Handle mode=filterValues and mode=ids for trustee positions.""" from modules.dbHelpers.paginationHelpers import handleIdsInMemory, handleFilterValuesInMemory from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels - from .datamodelFeatureTrustee import TrusteePositionView interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) if mode == "filterValues": if not column: @@ -1507,7 +1504,6 @@ def delete_accounting_config( """Remove the accounting integration for this instance.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import TrusteeAccountingConfig records = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId}) for r in records: interface.db.recordDelete(TrusteeAccountingConfig, r.get("id")) @@ -1602,7 +1598,6 @@ def get_sync_status( """Get sync status of all positions for this instance.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import TrusteeAccountingSync records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId}) return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]} @@ -1618,7 +1613,6 @@ def get_position_sync_status( """Get sync status for a specific position.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import TrusteeAccountingSync records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"positionId": positionId, "featureInstanceId": instanceId}) return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]} @@ -1776,7 +1770,6 @@ def _serializeRoleForApi(role) -> Dict[str, Any]: here (same pattern as ``getQuickActions``). Without this the React tree crashes with "Objects are not valid as a React child". """ - from modules.shared.i18nRegistry import resolveText payload = role.model_dump() payload["description"] = resolveText(payload.get("description")) return payload diff --git a/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py b/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py index 240809c1..9574b7c6 100644 --- a/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py +++ b/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py @@ -269,7 +269,6 @@ async def _extractWithAi( ) -> Dict[str, Any]: """3-step extraction: (1a) OCR/text via Vision AI, (1b) classify text, (2) structure by type.""" await self.services.ai.ensureAiObjectsInitialized() - from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference docList = DocumentReferenceList( references=[DocumentItemReference(documentId=chatDocumentId, fileName=fileName)] diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index c6d143dd..18cf0da8 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -917,7 +917,6 @@ async def _runWorkspaceAgent( messagePersisted = False _toolSet = _cfg.get("toolSet", "core") _agentCfg = _cfg.get("agentConfig") - from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentConfig agentCfgDict = dict(_agentCfg) if isinstance(_agentCfg, dict) else {} try: diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index a1800648..77bc1f5e 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -465,7 +465,6 @@ class AiObjects: toolChoice: Any = None, ) -> AsyncGenerator[Union[str, AiCallResponse], None]: """Stream a model call. Yields str deltas, then final AiCallResponse with billing.""" - from modules.datamodels.datamodelAi import AiModelCall, AiModelResponse inputBytes = sum(len(str(m.get("content", "")).encode("utf-8")) for m in messages) startTime = time.time() @@ -537,7 +536,6 @@ class AiObjects: Returns: AiCallResponse with metadata["embeddings"] containing the vectors. """ - from modules.aicore.aicoreBase import ContextLengthExceededException if options is None: options = AiCallOptions(operationType=OperationTypeEnum.EMBEDDING) diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index a3dda3ba..250190fb 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -220,7 +220,6 @@ class AppObjects: tableName = modelClass.__name__ # Use buildDataObjectKey for semantic namespace lookup - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName) permissions = self.rbac.getUserPermissions( self.currentUser, @@ -1122,8 +1121,6 @@ class AppObjects: def _deleteUserReferencedData(self, userId: str) -> None: """Deletes all data associated with a user (full cascade).""" try: - from modules.datamodels.datamodelNotification import UserNotification - from modules.datamodels.datamodelInvitation import Invitation # 1. FeatureAccess + FeatureAccessRole accesses = self.db.getRecordset(FeatureAccess, recordFilter={"userId": userId}) @@ -1560,7 +1557,6 @@ class AppObjects: # Copy system template roles to new mandate (admin, user, viewer + AccessRules) try: - from modules.interfaces.interfaceRbac import copySystemRolesToMandate copiedCount = copySystemRolesToMandate(self.db, mandateId) logger.info(f"Copied {copiedCount} system roles to new mandate {mandateId}") except Exception as e: @@ -1576,8 +1572,6 @@ class AppObjects: ``mandateLabel`` is the display name (Voller Name); a unique slug ``name`` (Kurzzeichen) is derived. """ from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS - from modules.datamodels.datamodelFeatures import FeatureInstance - from modules.interfaces.interfaceRbac import copySystemRolesToMandate from modules.interfaces.interfaceFeatures import getFeatureInterface from modules.shared.featureDiscovery import loadFeatureMainModules plan = BUILTIN_PLANS.get(planKey) @@ -1847,7 +1841,6 @@ class AppObjects: raise PermissionError(f"No permission to delete mandate {mandateId}") if not force: - from modules.shared.timeUtils import getUtcTimestamp self.db.recordModify(Mandate, mandateId, {"enabled": False, "deletedAt": getUtcTimestamp()}) logger.info(f"Soft-deleted mandate {mandateId} (30-day retention)") return True @@ -1858,8 +1851,6 @@ class AppObjects: from modules.datamodels.datamodelFiles import FileItem from modules.datamodels.datamodelDataSource import DataSource from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk - from modules.datamodels.datamodelFeatures import FeatureDataSource - from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) @@ -1983,7 +1974,6 @@ class AppObjects: # 3b. Billing data cascade handled by onMandateDelete lifecycle hook (interfaceDbBilling) # 3c. Delete Invitations for this mandate - from modules.datamodels.datamodelInvitation import Invitation invitations = self.db.getRecordset(Invitation, recordFilter={"mandateId": mandateId}) for inv in invitations: self.db.recordDelete(Invitation, inv.get("id")) @@ -1991,7 +1981,6 @@ class AppObjects: logger.info(f"Cascade: deleted {len(invitations)} Invitations for mandate {mandateId}") # 4. Delete mandate-level Roles - from modules.datamodels.datamodelRbac import Role, AccessRule roles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId}) for role in roles: rules = self.db.getRecordset(AccessRule, recordFilter={"roleId": role.get("id")}) @@ -3961,7 +3950,6 @@ class AppObjects: def getTableListViews(self, contextKey: str) -> list: """Return all saved views for the current user and contextKey.""" - from modules.datamodels.datamodelPagination import TableListView try: rows = self.db.getRecordset( TableListView, @@ -3980,7 +3968,6 @@ class AppObjects: def getTableListView(self, contextKey: str, viewKey: str): """Return one view by viewKey or None if not found.""" - from modules.datamodels.datamodelPagination import TableListView try: rows = self.db.getRecordset( TableListView, @@ -3996,8 +3983,6 @@ class AppObjects: def createTableListView(self, contextKey: str, viewKey: str, displayName: str, config: dict): """Create a new view. Raises ValueError if viewKey already exists for this context.""" - from modules.datamodels.datamodelPagination import TableListView - from modules.shared.timeUtils import getUtcTimestamp if self.getTableListView(contextKey=contextKey, viewKey=viewKey) is not None: raise ValueError(f"View '{viewKey}' already exists for context '{contextKey}'") data = { @@ -4018,8 +4003,6 @@ class AppObjects: def updateTableListView(self, viewId: str, updates: dict): """Update an existing view by its primary key id.""" - from modules.datamodels.datamodelPagination import TableListView - from modules.shared.timeUtils import getUtcTimestamp try: updates = {**updates, "updatedAt": getUtcTimestamp()} self.db.recordModify(TableListView, viewId, updates) @@ -4034,7 +4017,6 @@ class AppObjects: def deleteTableListView(self, viewId: str) -> bool: """Delete a view by primary key id. Returns True on success.""" - from modules.datamodels.datamodelPagination import TableListView try: self.db.recordDelete(TableListView, viewId) return True diff --git a/modules/interfaces/interfaceDbBilling.py b/modules/interfaces/interfaceDbBilling.py index 94600a0c..158f6d86 100644 --- a/modules/interfaces/interfaceDbBilling.py +++ b/modules/interfaces/interfaceDbBilling.py @@ -1654,8 +1654,6 @@ class BillingObjects: `amount` column. Resolves matching mandate/user IDs via the app DB first, then builds a single SQL query with OR-combined conditions. """ - from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields - from modules.datamodels.datamodelUam import UserInDB from modules.interfaces.interfaceDbApp import getInterface as getAppInterface table = BillingTransaction.__name__ diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index 71ccb774..4bbba04b 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -393,7 +393,6 @@ class ChatObjects: tableName = modelClass.__name__ # Use buildDataObjectKey for semantic namespace lookup - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName) permissions = self.rbac.getUserPermissions( self.currentUser, @@ -826,7 +825,6 @@ class ChatObjects: if not effectiveMandateId: # Fall back to Root mandate (first mandate in system) try: - from modules.datamodels.datamodelUam import Mandate from modules.security.rootAccess import getRootDbAppConnector dbAppConn = getRootDbAppConnector() allMandates = dbAppConn.getRecordset(Mandate) diff --git a/modules/interfaces/interfaceDbKnowledge.py b/modules/interfaces/interfaceDbKnowledge.py index 9c5a9bd3..dfafebe0 100644 --- a/modules/interfaces/interfaceDbKnowledge.py +++ b/modules/interfaces/interfaceDbKnowledge.py @@ -741,7 +741,6 @@ def migrateVectorDimensions(): If it differs from the target, nulls existing embeddings and alters the column type. Safe to call on every startup — skips when dimensions already match or table doesn't exist. """ - from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS targetDim = KNOWLEDGE_EMBEDDING_DIMENSIONS interface = getInterface() diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 93e2d1c3..66f6485c 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -317,7 +317,6 @@ class ComponentObjects: return False tableName = modelClass.__name__ - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey(tableName) permissions = self.rbac.getUserPermissions( self.currentUser, @@ -1066,7 +1065,6 @@ class ComponentObjects: Owners always can. Non-owners need RBAC ALL level.""" if self._isFolderOwner(folder): return - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey("FileFolder") permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, objectKey, @@ -1207,7 +1205,6 @@ class ComponentObjects: self._requireFolderWriteAccess(folder, folderId, "update") if scope == "global": - from modules.interfaces.interfaceRbac import buildDataObjectKey objectKey = buildDataObjectKey("FileFolder") permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, objectKey, @@ -1387,8 +1384,6 @@ class ComponentObjects: Owners always can. Non-owners need RBAC ALL level.""" if self._isFileOwner(file): return - from modules.interfaces.interfaceRbac import buildDataObjectKey - from modules.datamodels.datamodelRbac import AccessRuleContext objectKey = buildDataObjectKey("FileItem") permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, objectKey, diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index ebcf8c56..22047ab3 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -379,7 +379,6 @@ def getRecordsetWithRBAC( # Handle JSONB fields and ensure numeric types are correct # Import the helper function from connector module - from modules.connectors.connectorDbPostgre import getModelFields fields = getModelFields(modelClass) for record in records: for fieldName, fieldType in fields.items(): @@ -511,7 +510,6 @@ def getRecordsetPaginatedWithRBAC( whereValues.append(value) if pagination and pagination.filters: - from modules.connectors.connectorDbPostgre import getModelFields fields = getModelFields(modelClass) validColumns = set(fields.keys()) for key, val in pagination.filters.items(): @@ -545,7 +543,6 @@ def getRecordsetPaginatedWithRBAC( orderParts: List[str] = [] if pagination and pagination.sort: - from modules.connectors.connectorDbPostgre import getModelFields validColumns = set(getModelFields(modelClass).keys()) for sf in pagination.sort: if sf.field in validColumns: @@ -569,7 +566,6 @@ def getRecordsetPaginatedWithRBAC( cursor.execute(dataSql, whereValues) records = [dict(row) for row in cursor.fetchall()] - from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields fields = getModelFields(modelClass) for record in records: parseRecordFields(record, fields, f"table {table}") @@ -625,7 +621,6 @@ def getDistinctColumnValuesWithRBAC( if not connector._ensureTableExists(modelClass): return [] - from modules.connectors.connectorDbPostgre import getModelFields fields = getModelFields(modelClass) if column not in fields: return [] @@ -949,7 +944,6 @@ def buildRbacWhereClause( # Fall back to Root mandate (first mandate in system) for GROUP access # This allows system-level tables to be accessed without explicit mandate context try: - from modules.datamodels.datamodelUam import Mandate dbApp = getRootDbAppConnector() allMandates = dbApp.getRecordset(Mandate) if allMandates: diff --git a/modules/interfaces/interfaceTableHelpers.py b/modules/interfaces/interfaceTableHelpers.py index e7c188c5..84b4cdd0 100644 --- a/modules/interfaces/interfaceTableHelpers.py +++ b/modules/interfaces/interfaceTableHelpers.py @@ -85,7 +85,6 @@ def applyViewToParams(params: Optional["PaginationParams"], viewConfig: Optional Returns the (mutated) params, or a new minimal PaginationParams when params is None (so callers always get a valid object). """ - from modules.datamodels.datamodelPagination import SortField if not viewConfig: return params @@ -264,7 +263,6 @@ def buildGroupLayout( ------- (page_items, GroupLayout | None) """ - from modules.datamodels.datamodelPagination import GroupBand, GroupLayout if not groupByLevels: offset = (page - 1) * pageSize diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index e8daa385..ec6d7878 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -473,7 +473,6 @@ def list_feature_instances( items = [inst.model_dump() for inst in instances] from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels - from modules.datamodels.datamodelFeatures import FeatureInstance enrichRowsWithFkLabels(items, FeatureInstance, db=rootInterface.db) if mode == "filterValues": diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index 83aaef00..cd91f36c 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -40,7 +40,6 @@ def _getAdminMandateIds(context: RequestContext) -> List[str]: """Get mandate IDs where the user has an admin role.""" mandateIds = [] try: - from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() userMandates = rootInterface.getUserMandates(str(context.user.id)) for um in userMandates: @@ -64,7 +63,6 @@ def _getAdminMandateIds(context: RequestContext) -> List[str]: def _isRoleInAdminMandates(roleId: str, adminMandateIds: List[str]) -> bool: """Check if a role belongs to one of the admin's mandates.""" try: - from modules.interfaces.interfaceDbApp import getRootInterface rootInterface = getRootInterface() role = rootInterface.getRole(roleId) if not role: @@ -1405,7 +1403,6 @@ def cleanup_duplicate_access_rules( # Phase 2: Fix template role assignments # UserMandateRole should reference mandate-instance roles, not templates # ===================================================================== - from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole allUserMandateRoles = rootInterface.db.getRecordset(UserMandateRole) templateFixDetails = [] diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index ca95f7da..1d1441c4 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -756,7 +756,6 @@ def createOrUpdateSettings( return result or existingSettings return existingSettings else: - from modules.datamodels.datamodelBilling import BillingSettings newSettings = BillingSettings( mandateId=targetMandateId, @@ -821,7 +820,6 @@ def addCredit( if creditRequest.amount == 0: raise HTTPException(status_code=400, detail=routeApiMsg("Amount must not be zero")) - from modules.datamodels.datamodelBilling import BillingTransaction isDeduction = creditRequest.amount < 0 txType = TransactionTypeEnum.DEBIT if isDeduction else TransactionTypeEnum.CREDIT diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 2d425d25..b2830a40 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -161,7 +161,6 @@ async def get_connections( from modules.interfaces.interfaceTableHelpers import ( resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels, ) - from modules.datamodels.datamodelPagination import AppliedViewMeta CONTEXT_KEY = "connections" @@ -782,7 +781,6 @@ async def _updateKnowledgeConsent( if not connection: raise HTTPException(status_code=404, detail=routeApiMsg("Connection not found")) - from modules.interfaces.interfaceDbApp import getRootInterface rootIf = getRootInterface() rootIf.db.recordModify(UserConnection, connectionId, {"knowledgeIngestionEnabled": enabled}) @@ -861,7 +859,6 @@ def _updateKnowledgePreferences( cleaned = {k: v for k, v in preferences.items() if k in _ALLOWED_KEYS} merged = {**existing, **cleaned, "schemaVersion": 1} - from modules.interfaces.interfaceDbApp import getRootInterface getRootInterface().db.recordModify(UserConnection, connectionId, {"knowledgePreferences": merged}) logger.info("Knowledge preferences updated for connection %s", connectionId) diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 41625d26..cdb26f31 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -738,7 +738,6 @@ def get_files( resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels, ) import modules.interfaces.interfaceDbApp as _appIface - from modules.datamodels.datamodelPagination import AppliedViewMeta managementInterface = interfaceDbManagement.getInterface( currentUser, @@ -1202,7 +1201,6 @@ def bulk_set_neutralize( managementInterface.updateFile(fid, {"neutralize": neutralize}) if not neutralize: try: - from modules.interfaces import interfaceDbKnowledge kIface = interfaceDbKnowledge.getInterface(currentUser) kIface.purgeFileKnowledge(fid) except Exception as ke: diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index 164d4233..6f11493c 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -55,7 +55,6 @@ def get_prompts( resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels, ) from modules.interfaces.interfaceDbApp import getInterface as getAppInterface - from modules.datamodels.datamodelPagination import AppliedViewMeta CONTEXT_KEY = "prompts" diff --git a/modules/routes/routeMfa.py b/modules/routes/routeMfa.py index 5b13d592..7a2101ec 100644 --- a/modules/routes/routeMfa.py +++ b/modules/routes/routeMfa.py @@ -215,7 +215,6 @@ def mfaVerify( jti = jwt.decode(accessToken, SECRET_KEY, algorithms=[ALGORITHM]).get("jti") - from modules.interfaces.interfaceDbApp import getInterface user = User.model_validate(userRecord) userInterface = getInterface(user) dbToken = Token( diff --git a/modules/routes/routeNotifications.py b/modules/routes/routeNotifications.py index ef63fc1a..30adfef7 100644 --- a/modules/routes/routeNotifications.py +++ b/modules/routes/routeNotifications.py @@ -411,7 +411,6 @@ def _handleInvitationAction( ) -> str: """Handle accept/decline actions for invitation notifications.""" from modules.datamodels.datamodelInvitation import Invitation - from modules.datamodels.datamodelUam import Mandate from modules.datamodels.datamodelMembership import UserMandate invitationId = notification.referenceId diff --git a/modules/routes/routeRagInventory.py b/modules/routes/routeRagInventory.py index 82348d9a..9b2eee2e 100644 --- a/modules/routes/routeRagInventory.py +++ b/modules/routes/routeRagInventory.py @@ -485,7 +485,6 @@ def _getInventoryPlatform( from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface from modules.serviceCenter.services.serviceBackgroundJobs import mainBackgroundJobService as jobService - from modules.datamodels.datamodelUam import UserConnection rootIf = getRootInterface() knowledgeIf = getKnowledgeInterface(None) diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 3107a2b8..177cfaad 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -70,7 +70,6 @@ def buildAuthEmailHtml( operatorLine = "" try: - from modules.shared.configuration import APP_CONFIG parts = [p for p in [ APP_CONFIG.get("Operator_CompanyName", ""), APP_CONFIG.get("Operator_Address", ""), @@ -194,7 +193,6 @@ def _ensureHomeMandate(rootInterface, user) -> None: return try: - from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIf appIf = _getRootIf() normalizedEmail = (user.email or "").strip().lower() if user.email else None pendingByUsername = appIf.getInvitationsByTargetUsername(user.username) @@ -1058,7 +1056,6 @@ def _getNeutralizationMappings( ): """List the current user's neutralization placeholder mappings.""" userId = str(context.user.id) - from modules.interfaces.interfaceDbApp import getRootInterface from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes rootIf = getRootInterface() records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"userId": userId}) @@ -1074,7 +1071,6 @@ def _deleteNeutralizationMapping( ): """Delete a specific neutralization mapping owned by the current user.""" userId = str(context.user.id) - from modules.interfaces.interfaceDbApp import getRootInterface from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes rootIf = getRootInterface() records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"id": mappingId}) diff --git a/modules/routes/routeWorkflowAutomation.py b/modules/routes/routeWorkflowAutomation.py index 1ea2ba1b..c4e2fa84 100644 --- a/modules/routes/routeWorkflowAutomation.py +++ b/modules/routes/routeWorkflowAutomation.py @@ -64,7 +64,6 @@ async def _listWorkflows( mandateId: Optional[str] = Query(default=None), ): from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels - from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface db = _getWorkflowAutomationDb() try: db._ensureTableExists(AutoWorkflow) @@ -174,7 +173,6 @@ async def _listRuns( workflowId: Optional[str] = Query(default=None), ): from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels - from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface db = _getWorkflowAutomationDb() try: db._ensureTableExists(AutoRun) @@ -476,17 +474,14 @@ def _listTemplates( templates = iface.getTemplates(scope=scope) from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels - from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface enrichRowsWithFkLabels(templates, AutoWorkflow, db=_getRootIface().db) if mode == "filterValues": if not column: raise HTTPException(status_code=400, detail="column parameter required for mode=filterValues") - from modules.dbHelpers.paginationHelpers import handleFilterValuesInMemory return handleFilterValuesInMemory(templates, column, pagination) if mode == "ids": - from modules.dbHelpers.paginationHelpers import handleIdsInMemory return handleIdsInMemory(templates, pagination) paginationParams = None @@ -1328,7 +1323,6 @@ def _getRunDetail( if tid: try: from modules.dbHelpers.fkLabelResolver import resolveInstanceLabels - from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface labelMap = resolveInstanceLabels(_getRootIface().db, [tid]) targetInstanceLabel = labelMap.get(tid) except Exception: @@ -1425,7 +1419,6 @@ def _startEmailPollerIfNeeded(result: dict) -> None: if not isinstance(result, dict) or result.get("waitReason") != "email": return try: - from modules.interfaces.interfaceDbApp import getRootInterface from modules.workflowAutomation.scheduler.emailPoller import ensureRunning root = getRootInterface() eventUser = root.getUserByUsername("event") if root else None diff --git a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py index 81fc7f29..cbb576b2 100644 --- a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py +++ b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py @@ -425,7 +425,6 @@ class AgentService: activeToolNames.update(tb.tools) from modules.serviceCenter.services.serviceAgent.externalToolRegistry import getExternalTools - from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolDefinition for tb in activeToolboxes: extDefs = getExternalTools(tb.id) if not extDefs: @@ -459,7 +458,6 @@ class AgentService: from modules.serviceCenter.services.serviceAgent.toolboxRegistry import ( getToolboxRegistry, buildRequestToolboxDefinition, REQUEST_TOOLBOX_TOOL_NAME, ) - from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult tbRegistry = getToolboxRegistry() allIds = [tb.id for tb in tbRegistry.getAllToolboxes()] @@ -488,7 +486,6 @@ class AgentService: activatedCount += 1 continue try: - from modules.serviceCenter.services.serviceAgent.coreTools import registerCoreTools registerCoreTools(registry, self.services) if registry.isValidTool(toolName): activatedCount += 1 @@ -499,9 +496,6 @@ class AgentService: try: from modules.workflows.processing.shared.methodDiscovery import discoverMethods from modules.workflows.processing.core.actionExecutor import ActionExecutor - from modules.serviceCenter.services.serviceAgent.actionToolAdapter import ( - ActionToolAdapter, - ) discoverMethods(self.services) adapter = ActionToolAdapter(ActionExecutor(self.services)) @@ -622,7 +616,6 @@ class AgentService: def _createPersistRoundMemoryFn(self, workflowId: str): """Create callback that persists RoundMemory entries after tool execution.""" - from modules.serviceCenter.services.serviceAgent.agentLoop import classifyToolResult from modules.datamodels.datamodelKnowledge import RoundMemory async def _persistRoundMemory( diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index 79389b21..75fc77de 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -335,7 +335,6 @@ class AiService: Returns: AiCallResponse with content as JSON string (SpeechTeamsResponse format) """ - from modules.datamodels.datamodelAi import AiCallResponse, AiModelCall, AiCallOptions, PriorityEnum startTime = time.time() @@ -637,7 +636,6 @@ detectedIntent-Werte: try: from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector as _modSel - from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum _models = modelRegistry.getAvailableModels() _providers = self._calculateEffectiveProviders() diff --git a/modules/serviceCenter/services/serviceChat/mainServiceChat.py b/modules/serviceCenter/services/serviceChat/mainServiceChat.py index 44e42583..95a9248a 100644 --- a/modules/serviceCenter/services/serviceChat/mainServiceChat.py +++ b/modules/serviceCenter/services/serviceChat/mainServiceChat.py @@ -615,7 +615,6 @@ class ChatService: def getUserVoicePreferences(self, userId: str, mandateId: str = None) -> Optional[Dict[str, Any]]: """Get TTS voice preferences for a user, resolved by mandate scope.""" - from modules.datamodels.datamodelUam import UserVoicePreferences try: prefRecords = self.interfaceDbApp.db.getRecordset( UserVoicePreferences, recordFilter={"userId": userId} @@ -842,7 +841,6 @@ class ChatService: """Create an ActionItem record in the chat DB. Encapsulates low-level _separateObjectFields + db.recordCreate so callers never need direct interfaceDbChat access.""" - from modules.datamodels.datamodelChat import ActionItem simpleFields, _objectFields = self.interfaceDbChat._separateObjectFields(ActionItem, actionData) return self.interfaceDbChat.db.recordCreate(ActionItem, simpleFields) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py index 3e33db52..b71e6d65 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py @@ -204,7 +204,6 @@ def _addFilePart( entryPath = f"{containerPath}/{fileName}" if containerPath else fileName detectedMime = _detectMimeType(fileName) - from ..subRegistry import getExtractorRegistry registry = getExtractorRegistry() extractor = registry.resolve(detectedMime, fileName) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py index 6180f5d1..20a1fbd4 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py @@ -255,7 +255,6 @@ def _delegateAttachment(attachData: bytes, attachName: str, parentId: str, depth guessedMime, _ = mimetypes.guess_type(attachName) detectedMime = guessedMime or "application/octet-stream" - from ..subRegistry import getExtractorRegistry registry = getExtractorRegistry() extractor = registry.resolve(detectedMime, attachName) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py index 0f81fce0..fc11d6fe 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py @@ -141,7 +141,6 @@ def _walkFolder( guessedMime, _ = mimetypes.guess_type(entry.name) detectedMime = guessedMime or "application/octet-stream" - from ..subRegistry import ExtractorRegistry registry = ExtractorRegistry() extractor = registry.resolve(detectedMime, entry.name) diff --git a/modules/serviceCenter/services/serviceExtraction/subRegistry.py b/modules/serviceCenter/services/serviceExtraction/subRegistry.py index 7072ecbb..8f5be299 100644 --- a/modules/serviceCenter/services/serviceExtraction/subRegistry.py +++ b/modules/serviceCenter/services/serviceExtraction/subRegistry.py @@ -50,8 +50,6 @@ class Extractor: precomputedParts: Optional[List[ContentPart]] = None, ) -> "UdmDocument": """Build UDM from extracted parts (default: heuristic grouping). Override for format-specific trees.""" - from modules.datamodels.datamodelUdm import contentPartsToUdm, mimeToUdmSourceType - from modules.datamodels.datamodelExtraction import ContentExtracted from .subUtils import makeId parts = precomputedParts if precomputedParts is not None else self.extract(fileBytes, context) diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py index d1fe3b20..55fa8c99 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py @@ -230,7 +230,6 @@ class RendererPdf(BaseRenderer): # memory simultaneously. Collected here, deleted after the build. self._tempImageFiles = [] try: - from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle self._unifiedStyle = unifiedStyle or resolveStyle(None) styles = self._convertUnifiedStyleToInternal(self._unifiedStyle) for level in range(1, 7): diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py index 1547086f..4a1db42c 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py @@ -90,7 +90,6 @@ class RendererPptx(BaseRenderer): from pptx.dml.color import RGBColor if not style: - from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle style = resolveStyle(None) internalStyle = self._convertUnifiedStyleToInternal(style) styles = internalStyle diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py index aaa5d022..86846229 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py @@ -137,7 +137,6 @@ class RendererXlsx(BaseRenderer): self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(jsonContent.keys()) if isinstance(jsonContent, dict) else 'Not a dict'}", "EXCEL_RENDERER") if not style: - from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle style = resolveStyle(None) self._unifiedStyle = style diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py index 8e2f5935..edb347f1 100644 --- a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py +++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py @@ -26,6 +26,7 @@ from typing import Any, Callable, Dict, List, Optional from modules.serviceCenter.services.serviceKnowledge.subTextClean import cleanEmailBody from modules.serviceCenter.services.serviceKnowledge.subWalkerHelpers import ( WalkerTimeout, + extractWithTimeout as _extractWithTimeout, ingestWithTimeout, logItemStart, ) @@ -564,10 +565,6 @@ async def _ingestAttachments( attLabel = f"{messageId}/att:{stub['attachmentId']}/{fileName}" logItemStart("gmail-attachment", attLabel, sizeBytes=stub.get("size") or None, mime=mimeType) - from modules.serviceCenter.services.serviceKnowledge.subWalkerHelpers import ( - extractWithTimeout as _extractWithTimeout, - ) - def _runAttExtraction(): return runExtraction( extractorRegistry, chunkerRegistry, diff --git a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py index 439d9a5b..8714f660 100644 --- a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py +++ b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py @@ -414,7 +414,6 @@ class SubscriptionService: mandateLabel = mandateId try: - from modules.datamodels.datamodelUam import Mandate from modules.security.rootAccess import getRootDbAppConnector appDb = getRootDbAppConnector() rows = appDb.getRecordset(Mandate, recordFilter={"id": mandateId}) @@ -937,7 +936,6 @@ def _buildInvoiceSummaryHtml( ) -> str: """Build an HTML invoice summary block for inclusion in the activation email.""" import html as htmlmod - from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface subInterface = getSubRootInterface() userCount = subInterface.countActiveUsers(mandateId) diff --git a/modules/system/databaseHealth.py b/modules/system/databaseHealth.py index 5a9ec8fd..8b1ec19e 100644 --- a/modules/system/databaseHealth.py +++ b/modules/system/databaseHealth.py @@ -805,7 +805,6 @@ def _discoverLegacyTables(dbFilter: Optional[str] = None) -> List[dict]: Returns a list of dicts: {db, table, rowCount, sizeBytes}. """ from modules.datamodels.datamodelBase import MODEL_REGISTRY - from modules.dbHelpers.fkRegistry import ensureModelsLoaded ensureModelsLoaded() registeredDbs = getRegisteredDatabases() @@ -854,7 +853,6 @@ def _dropLegacyTable(dbName: str, tableName: str) -> dict: Raises ValueError if the table is model-backed (safety guard). """ from modules.datamodels.datamodelBase import MODEL_REGISTRY - from modules.dbHelpers.fkRegistry import ensureModelsLoaded ensureModelsLoaded() if tableName in MODEL_REGISTRY: diff --git a/modules/workflowAutomation/engine/executors/actionNodeExecutor.py b/modules/workflowAutomation/engine/executors/actionNodeExecutor.py index aa472f15..7e5cb30d 100644 --- a/modules/workflowAutomation/engine/executors/actionNodeExecutor.py +++ b/modules/workflowAutomation/engine/executors/actionNodeExecutor.py @@ -305,7 +305,6 @@ def _buildConnectionRefDict(connRef: str, chatService, services) -> Optional[Dic def _schemaCarriesConnectionProvenance(outputSchema: str) -> bool: """True iff the port schema declares ``carriesConnectionProvenance`` in the catalog.""" - from modules.nodeCatalog.portTypes import PORT_TYPE_CATALOG schema = PORT_TYPE_CATALOG.get(outputSchema) return bool(getattr(schema, "carriesConnectionProvenance", False)) diff --git a/modules/workflowAutomation/helpers.py b/modules/workflowAutomation/helpers.py index 9f28f274..a2121b7f 100644 --- a/modules/workflowAutomation/helpers.py +++ b/modules/workflowAutomation/helpers.py @@ -203,7 +203,6 @@ def _validateWorkflowAccess( if action == "execute": targetInstanceId = workflow.get("targetFeatureInstanceId") if targetInstanceId: - from modules.interfaces.interfaceDbApp import getRootInterface access = getRootInterface().getFeatureAccess(userId, targetInstanceId) if access and access.get("enabled"): return @@ -582,7 +581,6 @@ def _getWorkflowsJoinedPaginated( paginationParams: PaginationParams, ) -> dict: """SQL listing: AutoWorkflow LEFT JOIN aggregated AutoRun stats (one query + count).""" - from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields wfFields = getModelFields(AutoWorkflow) whereClause, orderClause, limitClause, values = _buildJoinedWorkflowWhereOrderLimit( diff --git a/modules/workflowAutomation/mainWorkflowAutomation.py b/modules/workflowAutomation/mainWorkflowAutomation.py index 086530c7..e07673c6 100644 --- a/modules/workflowAutomation/mainWorkflowAutomation.py +++ b/modules/workflowAutomation/mainWorkflowAutomation.py @@ -291,7 +291,6 @@ def onInstanceCreate(mandateId: str, instanceId: str, featureCode: str, template """Create workflow instances from template definitions when a feature instance is created.""" from modules.interfaces.interfaceWorkflowAutomation import _getWorkflowAutomationInterface from modules.security.rootAccess import getRootUser - from modules.shared.i18nRegistry import resolveText rootUser = getRootUser() waInterface = _getWorkflowAutomationInterface(rootUser, mandateId, instanceId) diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index 77adc40f..d943d7da 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -37,7 +37,6 @@ def _action_docs_to_content_parts(services, docs: List[Any]) -> List[ContentPart """Extract content from ActionDocument-like objects in memory (no persistence). Decodes base64, runs extraction pipeline, returns ContentParts for AI. """ - from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy all_parts = [] extraction = services.extraction @@ -78,7 +77,6 @@ def _resolve_file_refs_to_content_parts(services, fileIdRefs) -> List[ContentPar references, not chat message attachments. In the agent/chat context, ``DocumentItemReference`` holds ChatDocument IDs that must be resolved via ``getChatDocumentsFromDocumentList`` instead.""" - from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy extraction = services.extraction if not extraction: diff --git a/scripts/script_analyze_platform_module_graph.py b/scripts/script_analyze_platform_module_graph.py new file mode 100644 index 00000000..8e8866c4 --- /dev/null +++ b/scripts/script_analyze_platform_module_graph.py @@ -0,0 +1,568 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +""" +Deep platform-core module import graph analysis. + +Output: local/notes/refernce-analysis/import-analysis-platform-modules.md + +Usage: + python platform-core/scripts/script_analyze_platform_module_graph.py +""" + +from __future__ import annotations + +import ast +import os +import sys +from collections import defaultdict +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) + +from script_analyze_porta_imports import ( # noqa: E402 + OUTPUT_ROOT, + PLATFORM_ROOT, + SKIP_DIR_NAMES, + _collectPlatformModules, + _getPlatformContainer, + _platformModuleId, + _resolvePlatformImportTarget, + _resolvePlatformRelativeImport, + _writeText, +) + + +OUTPUT_FILE = OUTPUT_ROOT / "import-analysis-platform-modules.md" + +LAYER_ORDER = { + "shared": 0, + "datamodels": 1, + "connectors": 2, + "nodeCatalog": 2, + "dbHelpers": 3, + "interfaces": 4, + "system": 4, + "security": 4, + "auth": 4, + "aicore": 4, + "demoConfigs": 4, + "serviceCenter": 5, + "workflows": 5, + "workflowAutomation": 5, + "features.commcoach": 5, + "features.neutralization": 5, + "features.realEstate": 5, + "features.realestate": 5, + "features.redmine": 5, + "features.teamsbot": 5, + "features.trustee": 5, + "features.workspace": 5, + "routes": 6, + "app": 7, +} + + +@dataclass +class ScopedImport: + target: str + rawModule: str + position: str + scope: str + isInternal: bool + isStdLib: bool + + +def _shortModule(moduleId: str) -> str: + parts = moduleId.replace("platform-core.", "").split(".") + if len(parts) <= 3: + return ".".join(parts) + return ".".join(parts[-3:]) + + +LIFECYCLE_SCOPE_MARKERS = ( + "lifespan", + "onBootstrap", + "onStart", + "onStop", + "onInstanceCreate", + "onMandateDelete", + "registerFeature", + "preWarm", +) + + +def _layerOf(moduleId: str) -> Optional[int]: + container = _getPlatformContainer(moduleId) + if container is None: + return None + return LAYER_ORDER.get(container) + + +def _isStdLibModule(moduleName: str) -> bool: + root = moduleName.split(".")[0] + if root.startswith("_"): + return False + if root in sys.builtin_module_names: + return True + if hasattr(sys, "stdlib_module_names") and root in sys.stdlib_module_names: + return True + return False + + +class _DetailedImportVisitor(ast.NodeVisitor): + def __init__(self, filePath: Path): + self.filePath = filePath + self.imports: List[ScopedImport] = [] + self._scopeStack: List[str] = [] + + @property + def _currentScope(self) -> str: + return self._scopeStack[-1] if self._scopeStack else "" + + def _position(self) -> str: + return "code" if self._scopeStack else "header" + + def _add(self, rawModule: str, resolved: str, isInternal: bool) -> None: + self.imports.append( + ScopedImport( + target=resolved, + rawModule=rawModule, + position=self._position(), + scope=self._currentScope, + isInternal=isInternal, + isStdLib=_isStdLibModule(rawModule) if not isInternal else False, + ) + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._scopeStack.append(f"function {node.name}") + self.generic_visit(node) + self._scopeStack.pop() + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._scopeStack.append(f"function {node.name}") + self.generic_visit(node) + self._scopeStack.pop() + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._scopeStack.append(f"class {node.name}") + self.generic_visit(node) + self._scopeStack.pop() + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + resolved, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name) + self._add(alias.name, resolved, isInternal) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if node.level > 0: + resolved = _resolvePlatformRelativeImport(self.filePath, node) + if resolved: + suffix = node.module or "" + raw = ("." * node.level) + suffix + self._add(raw, resolved, True) + return + if not node.module: + return + resolved, isInternal = _resolvePlatformImportTarget(self.filePath, node.module) + self._add(node.module, resolved, isInternal) + + +def _collectDetailedImports() -> Dict[str, List[ScopedImport]]: + byModule: Dict[str, List[ScopedImport]] = {} + pyFiles: List[Path] = [] + appFile = PLATFORM_ROOT / "app.py" + if appFile.exists(): + pyFiles.append(appFile) + for root, dirs, files in os.walk(PLATFORM_ROOT / "modules"): + dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES] + for fileName in files: + if fileName.endswith(".py"): + pyFiles.append(Path(root) / fileName) + + for filePath in pyFiles: + moduleId = _platformModuleId(filePath) + if _getPlatformContainer(moduleId) is None: + continue + try: + tree = ast.parse(filePath.read_text(encoding="utf-8"), filename=str(filePath)) + except (SyntaxError, UnicodeDecodeError): + continue + visitor = _DetailedImportVisitor(filePath) + visitor.visit(tree) + byModule[moduleId] = visitor.imports + return byModule + + +def _internalGraph(importsByModule: Dict[str, List[ScopedImport]]) -> Dict[str, Set[str]]: + graph: Dict[str, Set[str]] = defaultdict(set) + for source, items in importsByModule.items(): + for item in items: + if item.isInternal and item.target.startswith("platform-core."): + graph[source].add(item.target) + return dict(graph) + + +def _mutualPairs( + graph: Dict[str, Set[str]], + moduleFilter: Optional[Set[str]] = None, +) -> List[Tuple[str, str]]: + pairs: List[Tuple[str, str]] = [] + seen: Set[Tuple[str, str]] = set() + sources = moduleFilter if moduleFilter is not None else set(graph.keys()) + for source in sorted(sources): + for target in graph.get(source, set()): + if moduleFilter is not None and target not in moduleFilter: + continue + if target not in graph or source not in graph[target]: + continue + key = tuple(sorted((source, target))) + if key not in seen: + seen.add(key) + pairs.append(key) + return pairs + + +def _tarjanScc(graph: Dict[str, Set[str]]) -> List[List[str]]: + index = 0 + stack: List[str] = [] + onStack: Set[str] = set() + indices: Dict[str, int] = {} + lowLink: Dict[str, int] = {} + result: List[List[str]] = [] + + nodes = set(graph.keys()) + for targets in graph.values(): + nodes.update(targets) + + def strongConnect(node: str) -> None: + nonlocal index + indices[node] = index + lowLink[node] = index + index += 1 + stack.append(node) + onStack.add(node) + + for neighbor in graph.get(node, set()): + if neighbor not in indices: + strongConnect(neighbor) + lowLink[node] = min(lowLink[node], lowLink[neighbor]) + elif neighbor in onStack: + lowLink[node] = min(lowLink[node], indices[neighbor]) + + if lowLink[node] == indices[node]: + component: List[str] = [] + while True: + w = stack.pop() + onStack.remove(w) + component.append(w) + if w == node: + break + if len(component) > 1 or (len(component) == 1 and component[0] in graph.get(component[0], set())): + result.append(sorted(component)) + + for node in sorted(nodes): + if node not in indices: + strongConnect(node) + return sorted(result, key=lambda c: (len(c), c[0]), reverse=True) + + +def _canReach(graph: Dict[str, Set[str]], start: str, goal: str, skipEdge: Optional[Tuple[str, str]] = None) -> bool: + visited: Set[str] = set() + + def dfs(node: str) -> bool: + if node == goal: + return True + if node in visited: + return False + visited.add(node) + for nxt in graph.get(node, set()): + if skipEdge and node == skipEdge[0] and nxt == skipEdge[1]: + continue + if dfs(nxt): + return True + return False + + return dfs(start) + + +def _assessMutualPair(a: str, b: str) -> str: + containerA = _getPlatformContainer(a) + containerB = _getPlatformContainer(b) + layerA = _layerOf(a) + layerB = _layerOf(b) + sameContainer = containerA == containerB + + if sameContainer: + if layerA is not None and layerA >= 5: + return "Prüfen — Feature/Service-interner Gegenimport; oft Lazy-Import-Workaround, Zyklus im Container." + return "Prüfen — gegenseitiger Import im gleichen Container; meist absichtlicher Lazy-Import gegen Zyklus." + + if layerA is not None and layerB is not None: + if layerA < layerB and layerB < layerA: + pass + upward = (layerA > layerB and layerB is not None) or (layerB > layerA and layerA is not None) + if upward: + return "Refactor-Kandidat — untere Schicht importiert obere und umgekehrt (Layer-Verletzung)." + return "Refactor-Kandidat — Cross-Container-Gegenimport; Layer-Grenze prüfen." + + +def _assessCycle(component: List[str]) -> str: + if len(component) == 1: + return "OK — Package-Reexport/Self-Import (__init__ ↔ Submodul); typisch für Barrel-Module." + containers = {_getPlatformContainer(m) for m in component} + containers.discard(None) + layers = [layer for m in component if (layer := _layerOf(m)) is not None] + if len(containers) == 1: + container = next(iter(containers)) + if LAYER_ORDER.get(container or "", 99) >= 5: + return "Prüfen — Zyklus innerhalb Feature/Service-Cluster; oft bekanntes Deferred-Coupling." + return "Prüfen — Intra-Container-Loop; Lazy-Imports prüfen ob extrahierbar." + if layers and max(layers) - min(layers) >= 2: + return "Refactor-Kandidat — Loop über mehrere Layer/Container; Architektur-Grenze verletzt." + return "Prüfen — Cross-Container-Loop; Abhängigkeit entkoppeln oder Typ/Protocol extrahieren." + + +def _assessLazyStdLib(moduleId: str, item: ScopedImport) -> str: + heavy = {"json", "csv", "xml", "pickle", "sqlite3", "subprocess", "multiprocessing"} + root = item.rawModule.split(".")[0] + if root in heavy: + return "OK — schwere Stdlib lazy (Startup/optional)." + if "TYPE_CHECKING" in item.scope: + return "OK — typing-only Kontext." + return "Harmlos — Stdlib lazy in Code-Scope; kein Architektur-Risiko." + + +def _assessMovable(moduleId: str, item: ScopedImport, graph: Dict[str, Set[str]], headerTargets: Set[str]) -> str: + if item.target in headerTargets: + return "Redundant — bereits im Header importiert; Lazy-Import entfernen." + if any(marker in item.scope for marker in LIFECYCLE_SCOPE_MARKERS): + return "Beabsichtigt lazy — Startup/Lifecycle-Hook; nicht in Header verschieben." + if _canReach(graph, item.target, moduleId): + return "Muss lazy bleiben — Header-Import würde Zyklus erzeugen." + return "Verschiebbar — kann vermutlich in den Header." + + +def _renderMarkdown( + importsByModule: Dict[str, List[ScopedImport]], + graph: Dict[str, Set[str]], +) -> str: + modulesByContainer: Dict[str, Set[str]] = defaultdict(set) + for moduleId in importsByModule: + container = _getPlatformContainer(moduleId) + if container: + modulesByContainer[container].add(moduleId) + + lines = [ + "# Import-Analyse Platform — Modul-Graph", + "", + f"- **Generiert:** {date.today().isoformat()}", + "- **Script:** `platform-core/scripts/script_analyze_platform_module_graph.py`", + "- **Scope:** interne `modules.*`-Imports (inkl. lazy)", + "", + "## Legende Beurteilung", + "", + "| Stufe | Bedeutung |", + "|-------|-----------|", + "| OK / Harmlos | kein Handlungsbedarf |", + "| Verschiebbar | Lazy-Import kann vermutlich in Header |", + "| Redundant | doppelter Import (Header + Code) |", + "| Prüfen | bekannt möglich, bewusst prüfen |", + "| Beabsichtigt lazy | Startup/Lifecycle — nicht in Header |", + "| Muss lazy bleiben | Zyklusvermeidung |", + "| Refactor-Kandidat | Layer-/Architektur-Thema |", + "", + ] + + # --- Mutual pairs per container --- + lines.extend(["## Gegenseitige Modul-Imports (Paare)", ""]) + totalPairs = 0 + for container in sorted(modulesByContainer.keys()): + moduleSet = modulesByContainer[container] + pairs = _mutualPairs(graph, moduleSet) + if not pairs: + continue + totalPairs += len(pairs) + lines.append(f"### Container `{container}`") + lines.append("") + lines.append("| Modul A | Modul B | Beurteilung |") + lines.append("|---------|---------|-------------|") + for a, b in pairs: + lines.append( + f"| `{_shortModule(a)}` | `{_shortModule(b)}` | {_assessMutualPair(a, b)} |" + ) + lines.append("") + + crossPairs = [ + p for p in _mutualPairs(graph) + if _getPlatformContainer(p[0]) != _getPlatformContainer(p[1]) + ] + if crossPairs: + lines.extend(["### Cross-Container (gegenseitig)", ""]) + lines.append("| Modul A | Container A | Modul B | Container B | Beurteilung |") + lines.append("|---------|-------------|---------|-------------|-------------|") + for a, b in crossPairs: + lines.append( + f"| `{_shortModule(a)}` | `{_getPlatformContainer(a)}` | " + f"`{_shortModule(b)}` | `{_getPlatformContainer(b)}` | {_assessMutualPair(a, b)} |" + ) + lines.append("") + if totalPairs == 0 and not crossPairs: + lines.append("_Keine gegenseitigen Modul-Paare gefunden._") + lines.append("") + + # --- Cycles --- + sccList = _tarjanScc(graph) + lines.extend(["## Import-Loops (über mehrere Module)", ""]) + if not sccList: + lines.append("_Keine Strongly-Connected Components (>1 Knoten) gefunden._") + lines.append("") + else: + lines.append(f"**{len(sccList)} Loop-Gruppe(n)** (Tarjan SCC, nur interne Module).") + lines.append("") + for index, component in enumerate(sccList, start=1): + containers = sorted({c for m in component if (c := _getPlatformContainer(m))}) + lines.append(f"### Loop {index} — {len(component)} Module") + lines.append("") + lines.append(f"- **Container:** {', '.join(f'`{c}`' for c in containers)}") + lines.append(f"- **Beurteilung:** {_assessCycle(component)}") + lines.append("- **Module:**") + for moduleId in component: + lines.append(f" - `{moduleId}`") + if len(component) <= 8: + chainHint = " → ".join(_shortModule(m) for m in component) + f" → `{_shortModule(component[0])}`" + lines.append(f"- **Ring (Auszug):** {chainHint}") + lines.append("") + + # --- Lazy stdlib --- + lines.extend(["## Lazy Stdlib-Imports (in Code-Scope)", ""]) + stdlibRows: List[Tuple[str, str, str, str, str]] = [] + for moduleId, items in sorted(importsByModule.items()): + for item in items: + if item.position == "code" and item.isStdLib: + stdlibRows.append( + ( + _shortModule(moduleId), + _getPlatformContainer(moduleId) or "", + item.rawModule, + item.scope or "(class/function)", + _assessLazyStdLib(moduleId, item), + ) + ) + if stdlibRows: + lines.append("| Modul | Container | Import | Scope | Beurteilung |") + lines.append("|-------|-----------|--------|-------|-------------|") + for row in stdlibRows: + lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |") + lines.append("") + else: + lines.append("_Keine lazy Stdlib-Imports in Code-Scope._") + lines.append("") + + # --- Lazy internal movable --- + lines.extend(["## Lazy interne Imports — Header möglich?", ""]) + movableRows: List[Tuple[str, str, str, str, str]] = [] + intentionalRows: List[Tuple[str, str, str, str, str]] = [] + mustStayRows: List[Tuple[str, str, str, str, str]] = [] + redundantRows: List[Tuple[str, str, str, str, str]] = [] + + for moduleId, items in sorted(importsByModule.items()): + headerTargets = {i.target for i in items if i.position == "header" and i.isInternal} + for item in items: + if item.position != "code" or not item.isInternal: + continue + verdict = _assessMovable(moduleId, item, graph, headerTargets) + row = ( + _shortModule(moduleId), + _getPlatformContainer(moduleId) or "", + _shortModule(item.target), + item.scope or "(code)", + verdict, + ) + if verdict.startswith("Verschiebbar"): + movableRows.append(row) + elif verdict.startswith("Beabsichtigt"): + intentionalRows.append(row) + elif verdict.startswith("Redundant"): + redundantRows.append(row) + elif verdict.startswith("Muss lazy"): + mustStayRows.append(row) + + if intentionalRows: + lines.append("### Beabsichtigt lazy (Startup/Lifecycle)") + lines.append("") + lines.append(f"**{len(intentionalRows)}** Einträge — lazy in lifespan/onBootstrap/…; kein Refactor nötig.") + lines.append("") + + if movableRows: + lines.append("### Verschiebbar in Header") + lines.append("") + lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |") + lines.append("|-------|-----------|-------------|-------|-------------|") + for row in movableRows: + lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |") + lines.append("") + + if mustStayRows: + lines.append("### Muss lazy bleiben (Zyklus)") + lines.append("") + lines.append(f"**{len(mustStayRows)}** Einträge — Auszug (max. 40):") + lines.append("") + lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |") + lines.append("|-------|-----------|-------------|-------|-------------|") + for row in mustStayRows[:40]: + lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |") + if len(mustStayRows) > 40: + lines.append("") + lines.append(f"_… und {len(mustStayRows) - 40} weitere._") + lines.append("") + + if redundantRows: + lines.append("### Redundant (Header + Code)") + lines.append("") + lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |") + lines.append("|-------|-----------|-------------|-------|-------------|") + for row in redundantRows: + lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |") + lines.append("") + + if not movableRows and not mustStayRows and not redundantRows and not intentionalRows: + lines.append("_Keine lazy internen Imports gefunden._") + lines.append("") + + lines.extend( + [ + "## Kurzfassung", + "", + f"- Gegenseitige Modul-Paare (intra-container): **{totalPairs}**", + f"- Gegenseitige Modul-Paare (cross-container): **{len(crossPairs)}**", + f"- Import-Loop-Gruppen (SCC): **{len(sccList)}** (davon Self-Loop: **{sum(1 for c in sccList if len(c) == 1)}**)", + f"- Lazy Stdlib-Imports: **{len(stdlibRows)}**", + f"- Lazy intern / beabsichtigt (Lifecycle): **{len(intentionalRows)}**", + f"- Lazy intern / verschiebbar: **{len(movableRows)}**", + f"- Lazy intern / Zyklus (muss bleiben): **{len(mustStayRows)}**", + f"- Lazy intern / redundant: **{len(redundantRows)}**", + "", + ] + ) + return "\n".join(lines) + + +def main() -> None: + print("Collecting detailed platform imports...") + importsByModule = _collectDetailedImports() + graph = _internalGraph(importsByModule) + print(f" modules: {len(importsByModule)}") + print(f" internal edges: {sum(len(v) for v in graph.values())}") + + markdown = _renderMarkdown(importsByModule, graph) + _writeText(OUTPUT_FILE, markdown) + print(f"Written: {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/scripts/script_analyze_porta_imports.py b/scripts/script_analyze_porta_imports.py new file mode 100644 index 00000000..c752d683 --- /dev/null +++ b/scripts/script_analyze_porta_imports.py @@ -0,0 +1,898 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +""" +Analyze all imports (including lazy/dynamic) for PowerOn PORTA UI and platform-core. + +Outputs under local/notes/refernce-analysis/: + platform/modules/*.md one file per Python module + platform/containers/*.md aggregated stats per container + platform/container-network.drawio + ui/modules/*.md + ui/containers/*.md + ui/container-network.drawio + README.md + +Usage: + python platform-core/scripts/script_analyze_porta_imports.py +""" + +from __future__ import annotations + +import ast +import html +import math +import os +import re +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Set, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +PLATFORM_ROOT = SCRIPT_DIR.parent +REPO_ROOT = PLATFORM_ROOT.parent +UI_ROOT = REPO_ROOT / "ui-nyla" +OUTPUT_ROOT = REPO_ROOT / "local" / "notes" / "refernce-analysis" + +SKIP_DIR_NAMES = { + "__pycache__", + "node_modules", + ".git", + "dist", + "build", + ".venv", + "venv", + ".tox", + ".mypy_cache", + ".pytest_cache", +} +UI_SKIP_GLOBS = ("**/*.test.ts", "**/*.test.tsx", "test/**") + + +@dataclass +class ImportRecord: + importedModule: str + position: str # "header" | "code" + isInternal: bool + sourceContainer: Optional[str] = None + targetContainer: Optional[str] = None + + +@dataclass +class ModuleAnalysis: + context: str # "platform" | "ui" + moduleId: str + filePath: Path + container: str + containerPath: str + imports: List[ImportRecord] = field(default_factory=list) + + +def _sanitizeFileName(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", value) + + +def _writeText(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Platform (Python) +# --------------------------------------------------------------------------- + +def _platformModuleId(filePath: Path) -> str: + rel = filePath.relative_to(PLATFORM_ROOT) + if filePath.name == "__init__.py": + parts = rel.parent.parts + else: + parts = rel.with_suffix("").parts + return "platform-core." + ".".join(parts) + + +def _platformContainerPath(container: str) -> str: + if container == "app": + return "platform-core/app.py" + if container.startswith("features."): + featureCode = container.split(".", 1)[1] + return f"platform-core/modules/features/{featureCode}" + return f"platform-core/modules/{container}" + + +def _getPlatformContainer(moduleId: str) -> Optional[str]: + if moduleId == "platform-core.app": + return "app" + + if not moduleId.startswith("platform-core."): + return None + + parts = moduleId.replace("platform-core.", "").split(".") + if not parts: + return "app" + + if parts[0] in ("tests", "scripts") or parts[0].startswith("script_"): + return None + if parts[0] != "modules" or len(parts) < 2: + return "app" + + container = parts[1] + if container == "features" and len(parts) > 2: + return f"features.{parts[2]}" + return container + + +def _resolvePlatformRelativeImport(currentFile: Path, importNode: ast.ImportFrom) -> Optional[str]: + dotCount = importNode.level + moduleSuffix = importNode.module or "" + currentDir = currentFile.parent + + baseDir = currentDir + for _ in range(dotCount - 1): + baseDir = baseDir.parent + + if moduleSuffix: + candidate = baseDir / Path(moduleSuffix.replace(".", os.sep)) + else: + candidate = baseDir + + pyFile = candidate.with_suffix(".py") + if pyFile.exists(): + return _platformModuleId(pyFile) + + initFile = candidate / "__init__.py" + if initFile.exists(): + return _platformModuleId(initFile) + + rel = candidate.relative_to(PLATFORM_ROOT) if candidate.is_relative_to(PLATFORM_ROOT) else None + if rel is None: + return None + return "platform-core." + ".".join(rel.with_suffix("").parts) + + +def _resolvePlatformImportTarget(currentFile: Path, importedName: str) -> Tuple[str, bool]: + if importedName.startswith("."): + return importedName, False + + if importedName.startswith("modules."): + parts = importedName.split(".") + checkPath = PLATFORM_ROOT + for part in parts: + checkPath = checkPath / part + if checkPath.with_suffix(".py").exists(): + return _platformModuleId(checkPath.with_suffix(".py")), True + if checkPath.is_dir() and (checkPath / "__init__.py").exists(): + return _platformModuleId(checkPath / "__init__.py"), True + return f"platform-core.{importedName.replace('.', '.')}", True + + return importedName, False + + +class _PythonImportVisitor(ast.NodeVisitor): + def __init__(self, filePath: Path): + self.filePath = filePath + self.imports: List[ImportRecord] = [] + self._inCodeScope = False + + def _addImport(self, importedModule: str, isInternal: bool) -> None: + position = "code" if self._inCodeScope else "header" + sourceContainer = _getPlatformContainer(_platformModuleId(self.filePath)) + targetContainer = _getPlatformContainer(importedModule) if isInternal else None + self.imports.append( + ImportRecord( + importedModule=importedModule, + position=position, + isInternal=isInternal, + sourceContainer=sourceContainer, + targetContainer=targetContainer, + ) + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + previous = self._inCodeScope + self._inCodeScope = True + self.generic_visit(node) + self._inCodeScope = previous + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + previous = self._inCodeScope + self._inCodeScope = True + self.generic_visit(node) + self._inCodeScope = previous + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + previous = self._inCodeScope + self._inCodeScope = True + self.generic_visit(node) + self._inCodeScope = previous + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + resolved, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name) + self._addImport(resolved, isInternal) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if node.level > 0: + resolved = _resolvePlatformRelativeImport(self.filePath, node) + if resolved: + self._addImport(resolved, True) + else: + suffix = node.module or "" + display = ("." * node.level) + suffix + self._addImport(f"(relative-unresolved) {display}", False) + return + + if not node.module: + return + + resolved, isInternal = _resolvePlatformImportTarget(self.filePath, node.module) + self._addImport(resolved, isInternal) + + +def _analyzePythonFile(filePath: Path) -> Optional[ModuleAnalysis]: + container = _getPlatformContainer(_platformModuleId(filePath)) + if container is None: + return None + + try: + source = filePath.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(filePath)) + except (SyntaxError, UnicodeDecodeError) as error: + print(f"WARN parse failed: {filePath}: {error}") + return None + + visitor = _PythonImportVisitor(filePath) + visitor.visit(tree) + + moduleId = _platformModuleId(filePath) + return ModuleAnalysis( + context="platform", + moduleId=moduleId, + filePath=filePath, + container=container, + containerPath=_platformContainerPath(container), + imports=visitor.imports, + ) + + +def _collectPlatformModules() -> List[ModuleAnalysis]: + modules: List[ModuleAnalysis] = [] + scanRoots = [PLATFORM_ROOT / "modules", PLATFORM_ROOT / "app.py"] + pyFiles: List[Path] = [] + if scanRoots[1].exists(): + pyFiles.append(scanRoots[1]) + for root, dirs, files in os.walk(scanRoots[0]): + dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES] + for fileName in files: + if fileName.endswith(".py"): + pyFiles.append(Path(root) / fileName) + + for filePath in pyFiles: + analysis = _analyzePythonFile(filePath) + if analysis: + modules.append(analysis) + return modules + + +# --------------------------------------------------------------------------- +# UI (TypeScript) +# --------------------------------------------------------------------------- + +TS_IMPORT_FROM_RE = re.compile( + r"""(?:^|\n)\s*(?:import|export)\s+(?:type\s+)?(?:[\w*\s{},\n\r]+?\sfrom\s+)?['"]([^'"]+)['"]""", + re.MULTILINE, +) +TS_SIDE_EFFECT_IMPORT_RE = re.compile( + r"""(?:^|\n)\s*import\s+['"]([^'"]+)['"]\s*;""", + re.MULTILINE, +) +TS_DYNAMIC_IMPORT_RE = re.compile(r"""import\s*\(\s*['"]([^'"]+)['"]\s*\)""") + + +def _uiModuleId(filePath: Path) -> str: + rel = filePath.relative_to(UI_ROOT / "src") + if filePath.name == "index.ts" or filePath.name == "index.tsx": + parts = rel.parent.parts + else: + parts = rel.with_suffix("").parts + return "ui-nyla.src." + ".".join(parts) + + +def _uiContainerPath(container: str) -> str: + if container.startswith("pages."): + suffix = container.split(".", 1)[1] + if suffix in ("admin", "basedata", "billing", "settings", "workflowAutomation"): + return f"ui-nyla/src/pages/{suffix}" + return f"ui-nyla/src/pages/views/{suffix}" + if container.startswith("components."): + suffix = container.split(".", 1)[1] + return f"ui-nyla/src/components/{suffix}" + return f"ui-nyla/src/{container}" + + +def _getUiContainer(moduleId: str) -> Optional[str]: + if not moduleId.startswith("ui-nyla.src."): + return None + + parts = moduleId.replace("ui-nyla.src.", "").split(".") + if not parts: + return None + + top = parts[0] + if top == "test": + return None + + if top == "pages": + if len(parts) >= 3 and parts[1] == "views": + return f"pages.{parts[2]}" + if len(parts) >= 2: + return f"pages.{parts[1]}" + return "pages" + + if top == "components" and len(parts) >= 2: + return f"components.{parts[1]}" + + return top + + +def _resolveUiImport(currentFile: Path, spec: str) -> Tuple[str, bool]: + if spec.startswith("."): + resolvedPath = (currentFile.parent / spec).resolve() + candidates = [ + resolvedPath, + resolvedPath.with_suffix(".ts"), + resolvedPath.with_suffix(".tsx"), + resolvedPath / "index.ts", + resolvedPath / "index.tsx", + ] + for candidate in candidates: + if candidate.exists() and candidate.is_relative_to(UI_ROOT / "src"): + return _uiModuleId(candidate), True + relDisplay = spec + return relDisplay, False + + return spec, False + + +def _findTsImportPosition(source: str, matchStart: int) -> str: + depth = 0 + inFunction = False + functionDepth = 0 + i = 0 + while i < matchStart: + char = source[i] + if char == "{": + depth += 1 + elif char == "}": + depth = max(0, depth - 1) + if inFunction and depth < functionDepth: + inFunction = False + i += 1 + + lookback = source[max(0, matchStart - 400):matchStart] + if re.search(r"(?:function\s*\w*\s*\(|=>\s*\{|(?:async\s+)?function\s+\w+\s*\()", lookback): + tail = lookback[lookback.rfind("\n") + 1:] + if "=>" in tail or "function" in tail: + bracePos = source.find("{", max(0, matchStart - 120), matchStart) + if bracePos >= 0: + return "code" + + return "header" if depth == 0 and not inFunction else "code" + + +def _analyzeTypeScriptFile(filePath: Path) -> Optional[ModuleAnalysis]: + container = _getUiContainer(_uiModuleId(filePath)) + if container is None: + return None + + try: + source = filePath.read_text(encoding="utf-8") + except UnicodeDecodeError as error: + print(f"WARN read failed: {filePath}: {error}") + return None + + imports: List[ImportRecord] = [] + seen: Set[Tuple[str, str, str]] = set() + + def _register(spec: str, position: str) -> None: + resolved, isInternal = _resolveUiImport(filePath, spec) + key = (resolved, position, spec) + if key in seen: + return + seen.add(key) + sourceContainer = container + targetContainer = _getUiContainer(resolved) if isInternal else None + imports.append( + ImportRecord( + importedModule=resolved, + position=position, + isInternal=isInternal, + sourceContainer=sourceContainer, + targetContainer=targetContainer, + ) + ) + + for match in TS_IMPORT_FROM_RE.finditer(source): + position = _findTsImportPosition(source, match.start()) + _register(match.group(1), position) + + for match in TS_SIDE_EFFECT_IMPORT_RE.finditer(source): + if match.group(1) in {m.group(1) for m in TS_IMPORT_FROM_RE.finditer(source)}: + continue + position = _findTsImportPosition(source, match.start()) + _register(match.group(1), position) + + for match in TS_DYNAMIC_IMPORT_RE.finditer(source): + position = _findTsImportPosition(source, match.start()) + _register(match.group(1), position) + + moduleId = _uiModuleId(filePath) + return ModuleAnalysis( + context="ui", + moduleId=moduleId, + filePath=filePath, + container=container, + containerPath=_uiContainerPath(container), + imports=imports, + ) + + +def _collectUiModules() -> List[ModuleAnalysis]: + srcRoot = UI_ROOT / "src" + modules: List[ModuleAnalysis] = [] + for filePath in srcRoot.rglob("*"): + if not filePath.is_file(): + continue + if filePath.suffix not in (".ts", ".tsx"): + continue + rel = filePath.relative_to(srcRoot).as_posix() + if rel.startswith("test/") or rel.endswith(".test.ts") or rel.endswith(".test.tsx"): + continue + analysis = _analyzeTypeScriptFile(filePath) + if analysis: + modules.append(analysis) + return modules + + +# --------------------------------------------------------------------------- +# Markdown output +# --------------------------------------------------------------------------- + +def _renderModuleMarkdown(module: ModuleAnalysis) -> str: + lines = [ + f"# Module Import Analysis: `{module.moduleId}`", + "", + f"- **Kontext:** {module.context}", + f"- **Container:** `{module.container}`", + f"- **Container-Pfad:** `{module.containerPath}`", + f"- **Datei:** `{module.filePath.relative_to(REPO_ROOT).as_posix()}`", + f"- **Import-Anzahl:** {len(module.imports)}", + "", + "## Imports", + "", + "| Modul | Position | Intern |", + "|-------|----------|--------|", + ] + + for item in sorted(module.imports, key=lambda x: (x.importedModule, x.position)): + internal = "ja" if item.isInternal else "nein" + lines.append(f"| `{item.importedModule}` | {item.position} | {internal} |") + + if not module.imports: + lines.append("| _keine_ | | |") + + lines.append("") + return "\n".join(lines) + + +@dataclass +class ContainerStats: + container: str + containerPath: str + importsFrom: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) + exportedTo: Dict[str, int] = field(default_factory=lambda: defaultdict(int)) + mixedWith: Dict[str, Tuple[int, int]] = field(default_factory=dict) + + +def _buildContainerStats(modules: Iterable[ModuleAnalysis]) -> Dict[str, ContainerStats]: + statsByContainer: Dict[str, ContainerStats] = {} + + for module in modules: + if module.container not in statsByContainer: + statsByContainer[module.container] = ContainerStats( + container=module.container, + containerPath=module.containerPath, + ) + + for item in module.imports: + if not item.isInternal: + continue + if not item.sourceContainer or not item.targetContainer: + continue + if item.sourceContainer == item.targetContainer: + continue + + stats = statsByContainer[item.sourceContainer] + stats.importsFrom[item.targetContainer] += 1 + + targetStats = statsByContainer.get(item.targetContainer) + if targetStats is None: + targetStats = ContainerStats( + container=item.targetContainer, + containerPath=_platformContainerPath(item.targetContainer) + if module.context == "platform" + else _uiContainerPath(item.targetContainer), + ) + statsByContainer[item.targetContainer] = targetStats + targetStats.exportedTo[item.sourceContainer] += 1 + + for containerName, stats in statsByContainer.items(): + mixed: Dict[str, Tuple[int, int]] = {} + for other, outCount in stats.importsFrom.items(): + inCount = stats.exportedTo.get(other, 0) + if inCount > 0: + mixed[other] = (outCount, inCount) + stats.mixedWith = mixed + + return statsByContainer + + +def _renderContainerMarkdown(context: str, stats: ContainerStats) -> str: + importsTotal = sum(stats.importsFrom.values()) + exportsTotal = sum(stats.exportedTo.values()) + mixedTotal = sum(min(pair[0], pair[1]) for pair in stats.mixedWith.values()) + + lines = [ + f"# Container Import Analysis: `{stats.container}`", + "", + f"- **Kontext:** {context}", + f"- **Container-Pfad:** `{stats.containerPath}`", + "", + "## Imports aus anderen Containern", + "", + f"- **Anzahl:** {importsTotal}", + f"- **Container ({len(stats.importsFrom)}):** " + + (", ".join(f"`{name}` ({count})" for name, count in sorted(stats.importsFrom.items())) or "_keine_"), + "", + "## Exports zu anderen Containern", + "", + f"- **Anzahl:** {exportsTotal}", + f"- **Container ({len(stats.exportedTo)}):** " + + (", ".join(f"`{name}` ({count})" for name, count in sorted(stats.exportedTo.items())) or "_keine_"), + "", + "## Cross (mixed Import/Export)", + "", + f"- **Anzahl bidirektionaler Paare:** {len(stats.mixedWith)}", + f"- **Mindest-Wechselzahl (min je Richtung):** {mixedTotal}", + ] + + if stats.mixedWith: + lines.extend(["", "| Container | Importe hinaus | Importe herein |", "|-----------|----------------|----------------|"]) + for other, (outCount, inCount) in sorted(stats.mixedWith.items()): + lines.append(f"| `{other}` | {outCount} | {inCount} |") + else: + lines.append("- **Container:** _keine_") + + lines.append("") + return "\n".join(lines) + + +def _renderReadme(platformModules: int, uiModules: int, platformContainers: int, uiContainers: int) -> str: + return f"""# PORTA Import-Analyse + +Generiert am {date.today().isoformat()} durch `platform-core/scripts/script_analyze_porta_imports.py`. + +## Umfang + +| Kontext | Module | Container | +|---------|--------|-----------| +| platform | {platformModules} | {platformContainers} | +| ui | {uiModules} | {uiContainers} | + +## Struktur + +- `import-analysis-platform.md` — konsolidierte Platform-Übersicht (Tabelle) +- `import-analysis-platform-modules.md` — Modul-Graph: Gegenimporte, Loops, lazy Imports +- `import-analysis-ui.md` — konsolidierte UI-Übersicht (Tabelle) +- `platform/modules/` — ein Markdown pro Python-Modul (alle Imports inkl. lazy) +- `platform/containers/` — aggregierte Container-Statistik +- `platform/container-network.drawio` — Container-Vernetzung (schwarz=einweg, rot=mixed) +- `ui/modules/` — ein Markdown pro TS/TSX-Modul +- `ui/containers/` — aggregierte Container-Statistik +- `ui/container-network.drawio` — Container-Vernetzung +- `container-network.drawio` — kombiniert (2 Diagramm-Tabs: platform + ui) + +## Position + +- `header` — Import auf Modulebene (Top-Level) +- `code` — Import innerhalb von Funktion/Klasse oder dynamisch (`import()`) + +## Regenerieren + +```bash +python platform-core/scripts/script_analyze_porta_imports.py +``` +""" + + +# --------------------------------------------------------------------------- +# draw.io +# --------------------------------------------------------------------------- + +CONTAINER_COLORS = { + "app": "#dae8fc", + "aicore": "#d5e8d4", + "auth": "#ffe6cc", + "connectors": "#e1d5e7", + "datamodels": "#fff2cc", + "interfaces": "#f8cecc", + "routes": "#d0cee2", + "security": "#fad7ac", + "serviceCenter": "#b1ddf0", + "shared": "#f0fff0", + "workflows": "#f5f5f5", + "workflowAutomation": "#e6d0de", + "system": "#cce5ff", + "dbHelpers": "#fff0f5", + "nodeCatalog": "#f5fffa", + "pages": "#dae8fc", + "components": "#d5e8d4", + "hooks": "#ffe6cc", + "contexts": "#e1d5e7", + "api": "#fff2cc", + "layouts": "#f8cecc", + "providers": "#d0cee2", + "config": "#fad7ac", + "utils": "#b1ddf0", + "types": "#f0fff0", + "locales": "#f5f5f5", + "stores": "#e2efda", + "styles": "#fce5cd", +} + + +def _aggregateContainerEdges(statsByContainer: Dict[str, ContainerStats]) -> Dict[Tuple[str, str], Tuple[int, int, bool]]: + pairCounts: Dict[Tuple[str, str], Tuple[int, int]] = {} + + for stats in statsByContainer.values(): + for target, count in stats.importsFrom.items(): + key = (stats.container, target) + outCount, inCount = pairCounts.get(key, (0, 0)) + pairCounts[key] = (outCount + count, inCount) + + edges: Dict[Tuple[str, str], Tuple[int, int, bool]] = {} + processed: Set[Tuple[str, str]] = set() + + for (source, target), (forward, _) in list(pairCounts.items()): + pairKey = tuple(sorted((source, target))) + if pairKey in processed: + continue + processed.add(pairKey) + + a, b = pairKey + aToB = pairCounts.get((a, b), (0, 0))[0] + bToA = pairCounts.get((b, a), (0, 0))[0] + + if aToB == 0 and bToA == 0: + continue + + if aToB > 0 and bToA > 0: + edges[(a, b)] = (aToB, bToA, True) + elif aToB > 0: + edges[(a, b)] = (aToB, 0, False) + else: + edges[(b, a)] = (bToA, 0, False) + + return edges + + +def _generateDrawio(context: str, statsByContainer: Dict[str, ContainerStats]) -> str: + containers = sorted(statsByContainer.keys()) + edges = _aggregateContainerEdges(statsByContainer) + + centerX = 700 + centerY = 550 + radius = 430 + nodeWidth = 170 + nodeHeight = 62 + + containerPositions: Dict[str, Tuple[int, int]] = {} + for index, container in enumerate(containers): + angle = (2 * math.pi * index / max(len(containers), 1)) - math.pi / 2 + x = int(centerX + radius * math.cos(angle) - nodeWidth / 2) + y = int(centerY + radius * math.sin(angle) - nodeHeight / 2) + containerPositions[container] = (x, y) + + cells: List[str] = [] + for container in containers: + x, y = containerPositions[container] + base = container.split(".")[0] + color = CONTAINER_COLORS.get(base, "#ffffff") + label = f"{container}\\n({sum(statsByContainer[container].importsFrom.values())} out / " + label += f"{sum(statsByContainer[container].exportedTo.values())} in)" + cellId = f"container_{container.replace('.', '_')}" + cells.append( + f""" + + """ + ) + + edgeId = 1000 + for (source, target), (forward, backward, isMixed) in sorted(edges.items(), key=lambda item: -(item[1][0] + item[1][1])): + sourceId = f"container_{source.replace('.', '_')}" + targetId = f"container_{target.replace('.', '_')}" + if isMixed: + label = f"{forward} / {backward}" + strokeColor = "#CC0000" + else: + label = str(forward) + strokeColor = "#000000" + + strokeWidth = min(1 + (forward + backward) // 15, 6) + cells.append( + f""" + + """ + ) + edgeId += 1 + + innerXml = f""" + + + +{chr(10).join(cells)} + + """ + + return _wrapDrawioDiagram(context, innerXml) + + +def _wrapDrawioDiagram(context: str, innerXml: str) -> str: + return f""" + + +{innerXml} + + +""" + + +def _extractDrawioDiagramBody(drawioXml: str) -> str: + start = drawioXml.index("") + return drawioXml[start:end] + + +def _combineDrawioFiles(platformDrawio: str, uiDrawio: str) -> str: + return f""" + + +{_extractDrawioDiagramBody(platformDrawio)} + + +{_extractDrawioDiagramBody(uiDrawio)} + + +""" + + +SUMMARY_FILE_PLATFORM = "import-analysis-platform.md" +SUMMARY_FILE_UI = "import-analysis-ui.md" + + +def _renderConsolidatedSummary( + title: str, + context: str, + detailFolder: str, + statsByContainer: Dict[str, ContainerStats], + diagramPath: str, +) -> str: + lines = [ + f"# {title}", + "", + f"- **Kontext:** {context}", + f"- **Generiert:** {date.today().isoformat()}", + f"- **Detail-Dateien:** `{detailFolder}/`", + "", + "## Container", + "", + "| Container | Imports out | Exports in | Mixed | Detail |", + "|-----------|------------:|-----------:|------:|--------|", + ] + + for containerName in sorted(statsByContainer.keys()): + stats = statsByContainer[containerName] + importsOut = sum(stats.importsFrom.values()) + exportsIn = sum(stats.exportedTo.values()) + mixedCount = len(stats.mixedWith) + detailLink = f"[Detail]({detailFolder}/containers/{_sanitizeFileName(containerName)}.md)" + lines.append( + f"| `{containerName}` | {importsOut} | {exportsIn} | {mixedCount} | {detailLink} |" + ) + + lines.extend( + [ + "", + f"Diagramm: [{diagramPath}]({diagramPath})", + "", + ] + ) + return "\n".join(lines) + + +def _writeContextOutput(context: str, modules: List[ModuleAnalysis]) -> Tuple[int, str]: + contextRoot = OUTPUT_ROOT / context + modulesDir = contextRoot / "modules" + containersDir = contextRoot / "containers" + + for module in modules: + fileName = _sanitizeFileName(module.moduleId) + ".md" + _writeText(modulesDir / fileName, _renderModuleMarkdown(module)) + + statsByContainer = _buildContainerStats(modules) + for containerName, stats in sorted(statsByContainer.items()): + fileName = _sanitizeFileName(containerName) + ".md" + _writeText(containersDir / fileName, _renderContainerMarkdown(context, stats)) + + drawio = _generateDrawio(context, statsByContainer) + _writeText(contextRoot / "container-network.drawio", drawio) + + return len(statsByContainer), drawio + + +def main() -> None: + print("Analyzing platform-core (Python)...") + platformModules = _collectPlatformModules() + print(f" modules: {len(platformModules)}") + + print("Analyzing ui-nyla (TypeScript)...") + uiModules = _collectUiModules() + print(f" modules: {len(uiModules)}") + + platformContainerCount, platformDrawio = _writeContextOutput("platform", platformModules) + uiContainerCount, uiDrawio = _writeContextOutput("ui", uiModules) + + combinedDrawio = _combineDrawioFiles(platformDrawio, uiDrawio) + _writeText(OUTPUT_ROOT / "container-network.drawio", combinedDrawio) + + readme = _renderReadme( + platformModules=len(platformModules), + uiModules=len(uiModules), + platformContainers=platformContainerCount, + uiContainers=uiContainerCount, + ) + _writeText(OUTPUT_ROOT / "README.md", readme) + + platformStats = _buildContainerStats(platformModules) + platformSummary = _renderConsolidatedSummary( + title="Import-Analyse Platform Core", + context="platform", + detailFolder="platform", + statsByContainer=platformStats, + diagramPath="platform/container-network.drawio", + ) + _writeText(OUTPUT_ROOT / SUMMARY_FILE_PLATFORM, platformSummary) + + uiStats = _buildContainerStats(uiModules) + uiSummary = _renderConsolidatedSummary( + title="Import-Analyse UI Nyla", + context="ui", + detailFolder="ui", + statsByContainer=uiStats, + diagramPath="ui/container-network.drawio", + ) + _writeText(OUTPUT_ROOT / SUMMARY_FILE_UI, uiSummary) + + print(f"\nOutput written to: {OUTPUT_ROOT}") + print(f" platform containers: {platformContainerCount}") + print(f" ui containers: {uiContainerCount}") + + +if __name__ == "__main__": + main() diff --git a/scripts/script_remove_redundant_platform_imports.py b/scripts/script_remove_redundant_platform_imports.py new file mode 100644 index 00000000..4081c7e4 --- /dev/null +++ b/scripts/script_remove_redundant_platform_imports.py @@ -0,0 +1,165 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +""" +Remove redundant lazy imports in platform-core when the same internal module +is already imported at module header level. + +Usage: + python platform-core/scripts/script_remove_redundant_platform_imports.py + python platform-core/scripts/script_remove_redundant_platform_imports.py --dry-run +""" + +from __future__ import annotations + +import argparse +import ast +import os +import sys +from pathlib import Path +from typing import Dict, List, Set, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) + +from script_analyze_porta_imports import ( # noqa: E402 + PLATFORM_ROOT, + SKIP_DIR_NAMES, + _getPlatformContainer, + _platformModuleId, + _resolvePlatformImportTarget, + _resolvePlatformRelativeImport, +) + + +class _RedundantImportFinder(ast.NodeVisitor): + def __init__(self, filePath: Path): + self.filePath = filePath + self.headerTargets: Set[str] = set() + self.linesToRemove: Set[int] = set() + self._scopeDepth = 0 + + def _resolveImportNode(self, node: ast.Import | ast.ImportFrom) -> List[Tuple[str, bool]]: + resolved: List[Tuple[str, bool]] = [] + if isinstance(node, ast.ImportFrom): + if node.level > 0: + target = _resolvePlatformRelativeImport(self.filePath, node) + if target: + resolved.append((target, True)) + return resolved + if not node.module: + return resolved + target, isInternal = _resolvePlatformImportTarget(self.filePath, node.module) + resolved.append((target, isInternal)) + return resolved + + for alias in node.names: + target, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name) + resolved.append((target, isInternal)) + return resolved + + def _handleImportNode(self, node: ast.Import | ast.ImportFrom) -> None: + for target, isInternal in self._resolveImportNode(node): + if not isInternal or not target.startswith("platform-core."): + continue + if self._scopeDepth == 0: + self.headerTargets.add(target) + elif target in self.headerTargets: + endLine = getattr(node, "end_lineno", None) or node.lineno + for lineNo in range(node.lineno, endLine + 1): + self.linesToRemove.add(lineNo) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self._scopeDepth += 1 + self.generic_visit(node) + self._scopeDepth -= 1 + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self._scopeDepth += 1 + self.generic_visit(node) + self._scopeDepth -= 1 + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._scopeDepth += 1 + self.generic_visit(node) + self._scopeDepth -= 1 + + def visit_Import(self, node: ast.Import) -> None: + self._handleImportNode(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + self._handleImportNode(node) + + +def _moduleIdToFilePath(moduleId: str) -> Path: + rel = moduleId.replace("platform-core.", "") + parts = rel.split(".") + candidate = PLATFORM_ROOT.joinpath(*parts).with_suffix(".py") + if candidate.exists(): + return candidate + initFile = PLATFORM_ROOT.joinpath(*parts, "__init__.py") + return initFile + + +def _collectPythonFiles() -> List[Path]: + pyFiles: List[Path] = [] + appFile = PLATFORM_ROOT / "app.py" + if appFile.exists(): + pyFiles.append(appFile) + for root, dirs, files in os.walk(PLATFORM_ROOT / "modules"): + dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES] + for fileName in files: + if fileName.endswith(".py"): + pyFiles.append(Path(root) / fileName) + return pyFiles + + +def _removeLines(filePath: Path, linesToRemove: Set[int], dryRun: bool) -> int: + if not linesToRemove: + return 0 + + lines = filePath.read_text(encoding="utf-8").splitlines(keepends=True) + newLines = [line for index, line in enumerate(lines, start=1) if index not in linesToRemove] + + if not dryRun: + filePath.write_text("".join(newLines), encoding="utf-8") + return len(linesToRemove) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + totalRemoved = 0 + filesChanged = 0 + details: List[Tuple[str, int]] = [] + + for filePath in _collectPythonFiles(): + moduleId = _platformModuleId(filePath) + if _getPlatformContainer(moduleId) is None: + continue + try: + tree = ast.parse(filePath.read_text(encoding="utf-8"), filename=str(filePath)) + except (SyntaxError, UnicodeDecodeError) as error: + print(f"WARN skip {filePath}: {error}") + continue + + finder = _RedundantImportFinder(filePath) + finder.visit(tree) + if not finder.linesToRemove: + continue + + removed = _removeLines(filePath, finder.linesToRemove, args.dry_run) + totalRemoved += removed + filesChanged += 1 + rel = filePath.relative_to(PLATFORM_ROOT.parent).as_posix() + details.append((rel, removed)) + action = "would remove" if args.dry_run else "removed" + print(f"{action} {removed} from {rel}") + + print(f"\nFiles touched: {filesChanged}") + print(f"Import lines {('would be ' if args.dry_run else '')}removed: {totalRemoved}") + + +if __name__ == "__main__": + main() From 29de7e9915874681a90a321d1653e506a3a05bfb Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 11 Jun 2026 23:28:49 +0200 Subject: [PATCH 5/8] fixes bad improrts by composer --- app.py | 2 +- modules/auth/trustedDeviceService.py | 2 +- modules/datamodels/datamodelRbac.py | 2 +- modules/features/commcoach/interfaceFeatureCommcoach.py | 3 +++ modules/features/commcoach/routeFeatureCommcoach.py | 7 +++++-- modules/features/commcoach/serviceCommcoach.py | 2 +- .../serviceNeutralization/subProcessList.py | 2 +- modules/features/realEstate/interfaceFeatureRealEstate.py | 2 +- modules/features/teamsbot/routeFeatureTeamsbot.py | 8 +++++++- modules/features/trustee/accounting/accountingBridge.py | 7 ++++++- modules/features/trustee/interfaceFeatureTrustee.py | 2 +- modules/features/trustee/routeFeatureTrustee.py | 2 +- modules/features/workspace/routeFeatureWorkspace.py | 2 +- modules/interfaces/interfaceAiObjects.py | 2 +- modules/interfaces/interfaceDbApp.py | 6 +++--- modules/interfaces/interfaceDbBilling.py | 4 ++-- modules/interfaces/interfaceDbChat.py | 4 ++-- modules/interfaces/interfaceDbKnowledge.py | 2 +- modules/interfaces/interfaceDbManagement.py | 2 +- modules/interfaces/interfaceRbac.py | 4 ++-- modules/interfaces/interfaceTableHelpers.py | 2 +- modules/routes/routeAdminRbacRules.py | 2 +- modules/routes/routeDataConnections.py | 4 ++-- modules/routes/routeDataFiles.py | 4 ++-- modules/routes/routeDataPrompts.py | 2 +- modules/routes/routeMfa.py | 2 +- modules/routes/routeRagInventory.py | 2 +- modules/routes/routeSecurityLocal.py | 2 +- modules/routes/routeWorkflowAutomation.py | 4 ++-- .../services/serviceAgent/mainServiceAgent.py | 4 ++-- modules/serviceCenter/services/serviceAi/mainServiceAi.py | 2 +- .../serviceCenter/services/serviceChat/mainServiceChat.py | 4 ++-- .../serviceExtraction/extractors/extractorContainer.py | 2 +- .../serviceExtraction/extractors/extractorEmail.py | 2 +- .../serviceExtraction/extractors/extractorFolder.py | 2 +- .../services/serviceExtraction/subRegistry.py | 3 ++- .../services/serviceGeneration/renderers/rendererPdf.py | 2 +- .../services/serviceGeneration/renderers/rendererPptx.py | 2 +- .../services/serviceGeneration/renderers/rendererXlsx.py | 2 +- .../serviceSubscription/mainServiceSubscription.py | 3 ++- modules/system/databaseHealth.py | 2 +- .../engine/executors/actionNodeExecutor.py | 1 + modules/workflowAutomation/helpers.py | 3 ++- modules/workflowAutomation/mainWorkflowAutomation.py | 2 +- modules/workflows/methods/methodAi/actions/process.py | 2 +- 45 files changed, 75 insertions(+), 54 deletions(-) diff --git a/app.py b/app.py index 1df8510f..2d1b9c41 100644 --- a/app.py +++ b/app.py @@ -21,7 +21,7 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager from modules.interfaces.interfaceDbApp import getRootInterface -from modules.system.registry import loadFeatureMainModules +from modules.system.registry import loadFeatureMainModules, registerAllFeaturesInCatalog, syncCatalogFeaturesToDb class DailyRotatingFileHandler(RotatingFileHandler): """ diff --git a/modules/auth/trustedDeviceService.py b/modules/auth/trustedDeviceService.py index d57d5c63..4ea85640 100644 --- a/modules/auth/trustedDeviceService.py +++ b/modules/auth/trustedDeviceService.py @@ -24,7 +24,7 @@ from fastapi import Request, Response from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcNow, getUtcTimestamp -from modules.datamodels.datamodelSecurity import TrustedDevice +from modules.datamodels.datamodelSecurity import TrustedDevice, Token, TokenPurpose logger = logging.getLogger(__name__) diff --git a/modules/datamodels/datamodelRbac.py b/modules/datamodels/datamodelRbac.py index 56196d84..4b2f953d 100644 --- a/modules/datamodels/datamodelRbac.py +++ b/modules/datamodels/datamodelRbac.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, Field from modules.datamodels.datamodelBase import PowerOnModel from modules.shared.i18nRegistry import i18nModel from modules.datamodels.datamodelUtils import TextMultilingual -from modules.datamodels.datamodelUam import AccessLevel +from modules.datamodels.datamodelUam import AccessLevel, User class AccessRuleContext(str, Enum): diff --git a/modules/features/commcoach/interfaceFeatureCommcoach.py b/modules/features/commcoach/interfaceFeatureCommcoach.py index a2e18770..689e3a92 100644 --- a/modules/features/commcoach/interfaceFeatureCommcoach.py +++ b/modules/features/commcoach/interfaceFeatureCommcoach.py @@ -23,6 +23,9 @@ from .datamodelCommcoach import ( CoachingTask, CoachingTaskStatus, CoachingScore, CoachingUserProfile, + CoachingPersona, + ModulePersonaMapping, + CoachingBadge, ) logger = logging.getLogger(__name__) diff --git a/modules/features/commcoach/routeFeatureCommcoach.py b/modules/features/commcoach/routeFeatureCommcoach.py index 905ffbc9..e8cf6fa4 100644 --- a/modules/features/commcoach/routeFeatureCommcoach.py +++ b/modules/features/commcoach/routeFeatureCommcoach.py @@ -18,7 +18,7 @@ from fastapi import APIRouter, HTTPException, Depends, Request, Query from fastapi.responses import StreamingResponse, Response from modules.auth import limiter, getRequestContext, RequestContext -from modules.shared.timeUtils import getIsoTimestamp +from modules.shared.timeUtils import getIsoTimestamp, getUtcTimestamp from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface @@ -33,7 +33,10 @@ from .datamodelCommcoach import ( UpdateProfileRequest, CreatePersonaRequest, UpdatePersonaRequest, SetModulePersonasRequest, ) -from .serviceCommcoach import CommcoachService, emitSessionEvent, getSessionEventQueue +from .serviceCommcoach import ( + CommcoachService, emitSessionEvent, getSessionEventQueue, + getUserVoicePrefs, stripMarkdownForTts, buildTtsConfigErrorMessage, +) from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeFeatureCommcoach") logger = logging.getLogger(__name__) diff --git a/modules/features/commcoach/serviceCommcoach.py b/modules/features/commcoach/serviceCommcoach.py index 315a9dff..5273c4ed 100644 --- a/modules/features/commcoach/serviceCommcoach.py +++ b/modules/features/commcoach/serviceCommcoach.py @@ -15,7 +15,7 @@ import asyncio from datetime import datetime, timezone from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, UserVoicePreferences from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum from modules.shared.timeUtils import getIsoTimestamp, getUtcTimestamp diff --git a/modules/features/neutralization/serviceNeutralization/subProcessList.py b/modules/features/neutralization/serviceNeutralization/subProcessList.py index ecc14b78..ac758a04 100644 --- a/modules/features/neutralization/serviceNeutralization/subProcessList.py +++ b/modules/features/neutralization/serviceNeutralization/subProcessList.py @@ -13,7 +13,7 @@ from typing import Dict, List, Any, Union from dataclasses import dataclass from io import StringIO from .subParseString import StringParser -from .subPatterns import getPatternForHeader, HeaderPatterns +from .subPatterns import getPatternForHeader, HeaderPatterns, findPatternsInText, DataPatterns @dataclass class NeutralizationTableData: diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 5d1ea3a6..cff30c04 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -29,7 +29,7 @@ from modules.dbHelpers.dbRegistry import registerDatabase from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC, buildDataObjectKey from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult logger = logging.getLogger(__name__) diff --git a/modules/features/teamsbot/routeFeatureTeamsbot.py b/modules/features/teamsbot/routeFeatureTeamsbot.py index c4219ac7..9b01fea5 100644 --- a/modules/features/teamsbot/routeFeatureTeamsbot.py +++ b/modules/features/teamsbot/routeFeatureTeamsbot.py @@ -49,7 +49,13 @@ from .datamodelTeamsbot import ( ) # Import service -from .service import TeamsbotService +from .service import ( + TeamsbotService, + getActiveService, + getActiveService as _getActiveService, + createAiService, + sessionEvents, +) from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeFeatureTeamsbot") diff --git a/modules/features/trustee/accounting/accountingBridge.py b/modules/features/trustee/accounting/accountingBridge.py index d9b9fe73..439a4ef4 100644 --- a/modules/features/trustee/accounting/accountingBridge.py +++ b/modules/features/trustee/accounting/accountingBridge.py @@ -19,7 +19,12 @@ from .accountingConnectorBase import ( SyncResult, ) from .accountingRegistry import getAccountingRegistry -from modules.features.trustee.datamodelFeatureTrustee import TrusteeDocument +from modules.features.trustee.datamodelFeatureTrustee import ( + TrusteeDocument, + TrusteeAccountingConfig, + TrusteePosition, + TrusteeAccountingSync, +) logger = logging.getLogger(__name__) diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index 249b2d50..6c8a8509 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -16,7 +16,7 @@ from pydantic import ValidationError from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG from modules.dbHelpers.dbRegistry import registerDatabase -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC, buildDataObjectKey from modules.security.rbac import RbacClass from modules.datamodels.datamodelUam import User, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 6e6b9924..3513e1da 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -49,7 +49,7 @@ from modules.datamodels.datamodelPagination import ( normalize_pagination_dict, ) from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext -from modules.shared.i18nRegistry import apiRouteContext +from modules.shared.i18nRegistry import apiRouteContext, resolveText routeApiMsg = apiRouteContext("routeFeatureTrustee") diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index 18cf0da8..fc95d57a 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -28,7 +28,7 @@ from modules.features.workspace import interfaceFeatureWorkspace from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface from modules.interfaces.interfaceAiObjects import AiObjects from modules.shared.eventManager import get_event_manager -from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum, PendingFileEdit +from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum, PendingFileEdit, AgentConfig from modules.shared.timeUtils import parseTimestamp from modules.shared.i18nRegistry import apiRouteContext, resolveText routeApiMsg = apiRouteContext("routeFeatureWorkspace") diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 77bc1f5e..b4eba2b2 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) from modules.aicore.aicoreModelRegistry import modelRegistry from modules.aicore.aicoreModelSelector import modelSelector -from modules.aicore.aicoreBase import RateLimitExceededException +from modules.aicore.aicoreBase import RateLimitExceededException, ContextLengthExceededException from modules.datamodels.datamodelAi import ( AiModel, AiCallOptions, diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 250190fb..e373461d 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -22,7 +22,7 @@ from modules.shared.configuration import APP_CONFIG from modules.dbHelpers.dbRegistry import registerDatabase from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp from modules.shared.i18nRegistry import resolveText -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, buildDataObjectKey, copySystemRolesToMandate from modules.security.rbac import RbacClass from modules.datamodels.datamodelUam import ( User, @@ -39,14 +39,14 @@ from modules.datamodels.datamodelRbac import ( ) from modules.datamodels.datamodelUam import AccessLevel from modules.datamodels.datamodelSecurity import Token, AuthEvent, TokenStatus, TokenPurpose -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult, TableListView from modules.datamodels.datamodelMembership import ( UserMandate, UserMandateRole, FeatureAccess, FeatureAccessRole, ) -from modules.datamodels.datamodelFeatures import Feature, FeatureInstance +from modules.datamodels.datamodelFeatures import Feature, FeatureInstance, FeatureDataSource, DataNeutralizerAttributes from modules.datamodels.datamodelInvitation import Invitation from modules.datamodels.datamodelNotification import UserNotification diff --git a/modules/interfaces/interfaceDbBilling.py b/modules/interfaces/interfaceDbBilling.py index 158f6d86..5b302dda 100644 --- a/modules/interfaces/interfaceDbBilling.py +++ b/modules/interfaces/interfaceDbBilling.py @@ -14,11 +14,11 @@ from typing import Dict, Any, List, Optional, Union from datetime import date, datetime, timedelta, timezone import uuid -from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.connectors.connectorDbPostgre import DatabaseConnector, getModelFields, parseRecordFields from modules.shared.configuration import APP_CONFIG from modules.dbHelpers.dbRegistry import registerDatabase from modules.shared.timeUtils import getUtcTimestamp -from modules.datamodels.datamodelUam import User, Mandate +from modules.datamodels.datamodelUam import User, Mandate, UserInDB from modules.datamodels.datamodelMembership import UserMandate from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.datamodels.datamodelBilling import ( diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index 4bbba04b..324f150e 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -27,14 +27,14 @@ from modules.datamodels.datamodelChat import ( UserInputRequest ) import json -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, Mandate # DYNAMIC PART: Connectors to the Interface from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.dbHelpers.dbRegistry import registerDatabase from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, buildDataObjectKey # Basic Configurations from modules.shared.configuration import APP_CONFIG diff --git a/modules/interfaces/interfaceDbKnowledge.py b/modules/interfaces/interfaceDbKnowledge.py index dfafebe0..ff7c7371 100644 --- a/modules/interfaces/interfaceDbKnowledge.py +++ b/modules/interfaces/interfaceDbKnowledge.py @@ -13,7 +13,7 @@ from typing import Dict, Any, List, Optional from modules.connectors.connectorDbPostgre import getCachedConnector from modules.dbHelpers.dbRegistry import registerDatabase -from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk, RoundMemory, WorkflowMemory +from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk, RoundMemory, WorkflowMemory, KNOWLEDGE_EMBEDDING_DIMENSIONS from modules.datamodels.datamodelUam import User from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcTimestamp diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 66f6485c..fc97483b 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -16,7 +16,7 @@ from typing import Dict, Any, List, Optional, Union from modules.connectors.connectorDbPostgre import DatabaseConnector, getCachedConnector from modules.dbHelpers.dbRegistry import registerDatabase -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, buildDataObjectKey from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 22047ab3..8556072b 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -32,10 +32,10 @@ from datetime import datetime, timezone from typing import List, Dict, Any, Optional, Type, Union from pydantic import BaseModel from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role -from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel +from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel, Mandate from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.datamodels.datamodelUtils import coerce_text_multilingual -from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.connectors.connectorDbPostgre import DatabaseConnector, getModelFields, parseRecordFields from modules.security.rbac import RbacClass from modules.security.rootAccess import getRootDbAppConnector diff --git a/modules/interfaces/interfaceTableHelpers.py b/modules/interfaces/interfaceTableHelpers.py index 84b4cdd0..0ad89ea0 100644 --- a/modules/interfaces/interfaceTableHelpers.py +++ b/modules/interfaces/interfaceTableHelpers.py @@ -12,7 +12,7 @@ from collections import defaultdict from functools import cmp_to_key from typing import Any, Dict, List, Optional -from modules.datamodels.datamodelPagination import PaginationParams +from modules.datamodels.datamodelPagination import PaginationParams, SortField, GroupBand, GroupLayout logger = logging.getLogger(__name__) diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index cd91f36c..6ce0398e 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -20,7 +20,7 @@ import math from modules.auth import limiter, getRequestContext, requirePlatformAdmin, RequestContext from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role -from modules.datamodels.datamodelMembership import UserMandate +from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.shared.i18nRegistry import apiRouteContext, t, resolveText diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index b2830a40..832b28fa 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -24,8 +24,8 @@ from modules.datamodels.datamodelSecurity import Token from modules.auth import getCurrentUser, limiter from modules.auth.oauthConnectTicket import issue_connect_ticket from modules.auth.tokenRefreshService import token_refresh_service -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.interfaces.interfaceDbApp import getInterface +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp from modules.interfaces.interfaceDbManagement import ComponentObjects from modules.shared.i18nRegistry import apiRouteContext diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index cdb26f31..745027bc 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -15,11 +15,11 @@ import zipfile from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext # Import interfaces -from modules.interfaces import interfaceDbManagement +from modules.interfaces import interfaceDbManagement, interfaceDbKnowledge from modules.datamodels.datamodelFiles import FileItem, FilePreview, FileFolder from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta from modules.shared.i18nRegistry import apiRouteContext from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels routeApiMsg = apiRouteContext("routeDataFiles") diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index 6f11493c..add6d580 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -16,7 +16,7 @@ from modules.auth import limiter, getCurrentUser from modules.interfaces import interfaceDbManagement from modules.datamodels.datamodelUtils import Prompt from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeDataPrompts") diff --git a/modules/routes/routeMfa.py b/modules/routes/routeMfa.py index 7a2101ec..6dafdf4e 100644 --- a/modules/routes/routeMfa.py +++ b/modules/routes/routeMfa.py @@ -32,7 +32,7 @@ from modules.auth.mfaService import ( verifyCode, isMfaRequired, ) -from modules.interfaces.interfaceDbApp import getRootInterface +from modules.interfaces.interfaceDbApp import getRootInterface, getInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate from modules.datamodels.datamodelSecurity import Token, TokenPurpose from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeRagInventory.py b/modules/routes/routeRagInventory.py index 9b2eee2e..4cc7437e 100644 --- a/modules/routes/routeRagInventory.py +++ b/modules/routes/routeRagInventory.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException, Depends, Request from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, UserConnection from modules.shared.i18nRegistry import apiRouteContext, resolveJobMessage routeApiMsg = apiRouteContext("routeRagInventory") diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 177cfaad..2b873e26 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -16,7 +16,7 @@ from jose import jwt # Import auth modules from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM, getRequestContext, RequestContext from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie -from modules.interfaces.interfaceDbApp import getInterface, getRootInterface +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface, getRootInterface as _getRootIf from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate from modules.datamodels.datamodelSecurity import Token, TokenPurpose from modules.shared.configuration import APP_CONFIG diff --git a/modules/routes/routeWorkflowAutomation.py b/modules/routes/routeWorkflowAutomation.py index c4e2fa84..619928f9 100644 --- a/modules/routes/routeWorkflowAutomation.py +++ b/modules/routes/routeWorkflowAutomation.py @@ -31,8 +31,8 @@ from modules.datamodels.datamodelPagination import PaginationParams, PaginationM from modules.datamodels.datamodelWorkflowAutomation import ( AutoWorkflow, AutoVersion, AutoRun, AutoStepLog, AutoTask, ) -from modules.dbHelpers.paginationHelpers import applyFiltersAndSort, paginateInMemory -from modules.interfaces.interfaceDbApp import getRootInterface +from modules.dbHelpers.paginationHelpers import applyFiltersAndSort, paginateInMemory, handleFilterValuesInMemory, handleIdsInMemory +from modules.interfaces.interfaceDbApp import getRootInterface, getRootInterface as _getRootIface from modules.shared.i18nRegistry import apiRouteContext, resolveText from modules.workflowAutomation.helpers import ( _getWorkflowAutomationDb, diff --git a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py index cbb576b2..1334b116 100644 --- a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py +++ b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py @@ -9,10 +9,10 @@ from modules.datamodels.datamodelAi import ( AiCallRequest, AiCallOptions, AiCallResponse, OperationTypeEnum ) from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( - AgentConfig, AgentEvent, AgentEventTypeEnum + AgentConfig, AgentEvent, AgentEventTypeEnum, ToolDefinition, ToolResult ) from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry -from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop +from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop, classifyToolResult from modules.serviceCenter.services.serviceAgent.actionToolAdapter import ActionToolAdapter from modules.serviceCenter.services.serviceAgent.coreTools import registerCoreTools import json diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index 75fc77de..622d6dd6 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -7,7 +7,7 @@ import time import base64 from typing import Dict, Any, List, Optional, Tuple, Callable from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument, WorkflowModeEnum -from modules.datamodels.datamodelAi import AiCallRequest, AiCallResponse, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.datamodels.datamodelAi import AiCallRequest, AiCallResponse, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum, AiModelCall from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData from modules.datamodels.datamodelDocument import RenderedDocument diff --git a/modules/serviceCenter/services/serviceChat/mainServiceChat.py b/modules/serviceCenter/services/serviceChat/mainServiceChat.py index 95a9248a..780c3c08 100644 --- a/modules/serviceCenter/services/serviceChat/mainServiceChat.py +++ b/modules/serviceCenter/services/serviceChat/mainServiceChat.py @@ -3,8 +3,8 @@ """Chat service for document processing, chat operations, and workflow management.""" import logging from typing import Dict, Any, List, Optional, Callable -from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatLog +from modules.datamodels.datamodelUam import User, UserConnection, UserVoicePreferences +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatLog, ActionItem from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger import json diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py index b71e6d65..c02c5405 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py @@ -22,7 +22,7 @@ import tarfile from ..subUtils import makeId from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelContent import ContainerLimitError, ContentContextRef -from ..subRegistry import Extractor +from ..subRegistry import Extractor, getExtractorRegistry import base64 logger = logging.getLogger(__name__) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py index 20a1fbd4..165ddede 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py @@ -18,7 +18,7 @@ import mimetypes from modules.datamodels.datamodelExtraction import ContentPart from ..subUtils import makeId -from ..subRegistry import Extractor +from ..subRegistry import Extractor, getExtractorRegistry import base64 logger = logging.getLogger(__name__) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py index fc11d6fe..c7ea525f 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py @@ -16,7 +16,7 @@ from pathlib import Path from ..subUtils import makeId from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelContent import ContainerLimitError, ContentContextRef -from ..subRegistry import Extractor +from ..subRegistry import Extractor, ExtractorRegistry logger = logging.getLogger(__name__) diff --git a/modules/serviceCenter/services/serviceExtraction/subRegistry.py b/modules/serviceCenter/services/serviceExtraction/subRegistry.py index 8f5be299..f4d59cd4 100644 --- a/modules/serviceCenter/services/serviceExtraction/subRegistry.py +++ b/modules/serviceCenter/services/serviceExtraction/subRegistry.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, TYPE_CHECKING import logging -from modules.datamodels.datamodelExtraction import ContentPart +from modules.datamodels.datamodelExtraction import ContentPart, ContentExtracted import os import traceback from pathlib import Path @@ -51,6 +51,7 @@ class Extractor: ) -> "UdmDocument": """Build UDM from extracted parts (default: heuristic grouping). Override for format-specific trees.""" from .subUtils import makeId + from modules.datamodels.datamodelUdm import contentPartsToUdm, mimeToUdmSourceType parts = precomputedParts if precomputedParts is not None else self.extract(fileBytes, context) eid = context.get("extractionId") or makeId() diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py index 55fa8c99..f30d6277 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py @@ -28,7 +28,7 @@ except ImportError: import re from ._pdfFontFallback import wrapEmojiSpansInXml as _wrapEmojiSpansInXml -from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge +from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle import os import tempfile diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py index 4a1db42c..53cf1d72 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py @@ -8,7 +8,7 @@ from datetime import datetime, UTC from typing import Dict, Any, Optional, List from .documentRendererBaseTemplate import BaseRenderer from modules.datamodels.datamodelDocument import RenderedDocument -from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge +from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle logger = logging.getLogger(__name__) diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py index 86846229..958dfb87 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py @@ -6,7 +6,7 @@ Excel renderer for report generation using openpyxl. from .documentRendererBaseTemplate import BaseRenderer from modules.datamodels.datamodelDocument import RenderedDocument -from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge +from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle from typing import Dict, Any, List, Optional import io import base64 diff --git a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py index 8714f660..feff8ae0 100644 --- a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py +++ b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py @@ -12,7 +12,7 @@ import time from typing import Dict, Any, List, Optional from datetime import datetime, timezone, timedelta -from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelUam import User, Mandate from modules.datamodels.datamodelSubscription import ( SubscriptionPlan, MandateSubscription, @@ -24,6 +24,7 @@ from modules.datamodels.datamodelSubscription import ( ) from modules.interfaces.interfaceDbSubscription import ( getInterface as getSubscriptionInterface, + getRootInterface as getSubRootInterface, InvalidTransitionError, ) from modules.shared.i18nRegistry import t diff --git a/modules/system/databaseHealth.py b/modules/system/databaseHealth.py index 8b1ec19e..c1027e17 100644 --- a/modules/system/databaseHealth.py +++ b/modules/system/databaseHealth.py @@ -20,7 +20,7 @@ import psycopg2.extras from modules.shared.configuration import APP_CONFIG from modules.dbHelpers.dbRegistry import getRegisteredDatabases -from modules.dbHelpers.fkRegistry import getFkRelationships, FkRelationship +from modules.dbHelpers.fkRegistry import getFkRelationships, FkRelationship, ensureModelsLoaded logger = logging.getLogger(__name__) diff --git a/modules/workflowAutomation/engine/executors/actionNodeExecutor.py b/modules/workflowAutomation/engine/executors/actionNodeExecutor.py index 7e5cb30d..b26c8251 100644 --- a/modules/workflowAutomation/engine/executors/actionNodeExecutor.py +++ b/modules/workflowAutomation/engine/executors/actionNodeExecutor.py @@ -17,6 +17,7 @@ from typing import Any, Dict, Optional from modules.nodeCatalog.portTypes import ( _normalizeError, normalizeToSchema, + PORT_TYPE_CATALOG, ) from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError from modules.workflows.methods.methodContext.actions.extractContent import ( diff --git a/modules/workflowAutomation/helpers.py b/modules/workflowAutomation/helpers.py index a2121b7f..ead4566a 100644 --- a/modules/workflowAutomation/helpers.py +++ b/modules/workflowAutomation/helpers.py @@ -24,7 +24,8 @@ from modules.datamodels.datamodelWorkflowAutomation import ( AutoRun, AutoStepLog, AutoWorkflow, AutoTask, AutoVersion, WORKFLOW_AUTOMATION_DATABASE, ) -from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface +from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface, getRootInterface +from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/workflowAutomation/mainWorkflowAutomation.py b/modules/workflowAutomation/mainWorkflowAutomation.py index e07673c6..5102d1e6 100644 --- a/modules/workflowAutomation/mainWorkflowAutomation.py +++ b/modules/workflowAutomation/mainWorkflowAutomation.py @@ -12,7 +12,7 @@ import logging import uuid from typing import Dict, List, Any, Optional -from modules.shared.i18nRegistry import t +from modules.shared.i18nRegistry import t, resolveText logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index d943d7da..56e97f8e 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -9,7 +9,7 @@ import uuid from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum -from modules.datamodels.datamodelExtraction import ContentPart +from modules.datamodels.datamodelExtraction import ContentPart, ExtractionOptions, MergeStrategy from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) From 66b44e5c78a7b91597004c0ce6efbd9a4e0b9385 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 12 Jun 2026 00:35:52 +0200 Subject: [PATCH 6/8] dead endpoint cleanup + admin sessions UX: remove unused endpoints (billing/balances, files/attributes, files/bulk, voice-google duplicates, workflow-automation/runs, jobs list), fix billing transaction filter for enriched columns, admin sessions use revokeTokenById instead of delete, add per-device revoke endpoint, add sessions nav item Co-authored-by: Cursor --- modules/datamodels/datamodelNavigation.py | 10 + modules/routes/routeAdminSessions.py | 52 +++-- modules/routes/routeBilling.py | 246 +++------------------- modules/routes/routeDataFiles.py | 229 -------------------- modules/routes/routeJobs.py | 32 +-- modules/routes/routeVoiceGoogle.py | 68 +----- modules/routes/routeWorkflowAutomation.py | 21 -- 7 files changed, 81 insertions(+), 577 deletions(-) diff --git a/modules/datamodels/datamodelNavigation.py b/modules/datamodels/datamodelNavigation.py index 101cef99..977e801c 100644 --- a/modules/datamodels/datamodelNavigation.py +++ b/modules/datamodels/datamodelNavigation.py @@ -320,6 +320,16 @@ NAVIGATION_SECTIONS = [ "adminOnly": True, "sysAdminOnly": True, }, + { + "id": "admin-sessions", + "objectKey": "ui.admin.sessions", + "label": t("Sessions & Geräte"), + "icon": "FaDesktop", + "path": "/admin/sessions", + "order": 92, + "adminOnly": True, + "sysAdminOnly": True, + }, { "id": "admin-languages", "objectKey": "ui.admin.languages", diff --git a/modules/routes/routeAdminSessions.py b/modules/routes/routeAdminSessions.py index d962a9ea..7503cd59 100644 --- a/modules/routes/routeAdminSessions.py +++ b/modules/routes/routeAdminSessions.py @@ -81,23 +81,28 @@ def revokeSession( sessionId: str, currentUser: User = Depends(getCurrentUser), ) -> Dict[str, Any]: - """Revoke a single session by sessionId.""" + """Revoke a single session by sessionId (sets status=REVOKED, not delete).""" _requireAdmin(currentUser) rootInterface = getRootInterface() + adminId = str(currentUser.id) tokens = rootInterface.db.getRecordset( Token, - recordFilter={"sessionId": sessionId, "tokenPurpose": TokenPurpose.AUTH_SESSION.value}, + recordFilter={ + "sessionId": sessionId, + "tokenPurpose": TokenPurpose.AUTH_SESSION.value, + "status": TokenStatus.ACTIVE.value, + }, ) count = 0 for t in tokens: - rootInterface.db.recordDelete(Token, t["id"]) + rootInterface.revokeTokenById(t["id"], revokedBy=adminId, reason="admin session revoke") count += 1 if count == 0: raise HTTPException(status_code=404, detail=routeApiMsg("Session not found")) - logger.info(f"Admin {currentUser.username} revoked session {sessionId} ({count} token(s))") + logger.info("Admin %s revoked session %s (%d token(s))", currentUser.username, sessionId, count) return {"revoked": count, "sessionId": sessionId} @@ -111,20 +116,13 @@ def revokeAllSessions( """Revoke ALL active sessions for a user (force logout everywhere).""" _requireAdmin(currentUser) rootInterface = getRootInterface() + adminId = str(currentUser.id) - tokens = rootInterface.db.getRecordset( - Token, - recordFilter={ - "userId": userId, - "tokenPurpose": TokenPurpose.AUTH_SESSION.value, - }, + count = rootInterface.revokeTokensByUser( + userId, revokedBy=adminId, reason="admin revoke all sessions", ) - count = 0 - for t in tokens: - rootInterface.db.recordDelete(Token, t["id"]) - count += 1 - logger.info(f"Admin {currentUser.username} revoked all sessions for userId={userId} ({count} token(s))") + logger.info("Admin %s revoked all sessions for userId=%s (%d token(s))", currentUser.username, userId, count) return {"revoked": count, "userId": userId} @@ -156,7 +154,7 @@ def listTrustedDevices( result = [] for d in devices: result.append({ - "id": d.get("id", "")[:8] + "...", + "id": d.get("id", ""), "trustedUntil": d.get("trustedUntil"), "isExpired": d.get("trustedUntil", 0) < now, "userAgent": d.get("userAgent"), @@ -167,6 +165,26 @@ def listTrustedDevices( return result +@trustedDeviceRouter.delete("/{deviceId}") +@limiter.limit("30/minute") +def revokeTrustedDevice( + request: Request, + deviceId: str, + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Revoke a single trusted device by ID.""" + _requireAdmin(currentUser) + rootInterface = getRootInterface() + + existing = rootInterface.db.getRecord(TrustedDevice, deviceId) + if not existing: + raise HTTPException(status_code=404, detail=routeApiMsg("Trusted device not found")) + + rootInterface.db.recordDelete(TrustedDevice, deviceId) + logger.info("Admin %s revoked trusted device %s", currentUser.username, deviceId) + return {"revoked": 1, "deviceId": deviceId} + + @trustedDeviceRouter.delete("") @limiter.limit("10/minute") def revokeAllTrustedDevices( @@ -181,5 +199,5 @@ def revokeAllTrustedDevices( from modules.auth.trustedDeviceService import revokeTrustedDevices count = revokeTrustedDevices(userId, rootInterface.db) - logger.info(f"Admin {currentUser.username} revoked all trusted devices for userId={userId} ({count})") + logger.info("Admin %s revoked all trusted devices for userId=%s (%d)", currentUser.username, userId, count) return {"revoked": count, "userId": userId} diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index 1d1441c4..84dc5c7b 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -290,19 +290,6 @@ class MandateBalanceResponse(BaseModel): warningThresholdPercent: float -class UserBalanceResponse(BaseModel): - """User-level balance summary.""" - accountId: str - mandateId: str - mandateName: str - userId: str - userName: str - balance: float - warningThreshold: float - isWarning: bool - enabled: bool - - class UserTransactionResponse(BaseModel): """User-level transaction with user context.""" id: str @@ -429,10 +416,6 @@ def _normalize_billing_tx_dict(t: Dict[str, Any]) -> Dict[str, Any]: return r -def _load_billing_user_transactions_normalized(billingService) -> List[Dict[str, Any]]: - raw = billingService.getTransactionHistory(limit=5000) - return [_normalize_billing_tx_dict(t) for t in raw] - def _view_user_transactions_filtered_list( billing_interface, @@ -464,147 +447,6 @@ def _view_user_transactions_filtered_list( return all_items -@router.get("/transactions") -@limiter.limit("30/minute") -def getTransactions( - request: Request, - limit: int = Query(default=50, ge=1, le=500), - offset: int = Query(default=0, ge=0), - pagination: Optional[str] = Query( - None, - description="JSON PaginationParams for table UI (filters, sort, viewKey, groupByLevels).", - ), - mode: Optional[str] = Query(None, description="'filterValues' | 'ids' with pagination"), - column: Optional[str] = Query(None, description="Column for mode=filterValues"), - ctx: RequestContext = Depends(getRequestContext), -): - """ - Get transaction history across all mandates the user belongs to. - - Without ``pagination`` query: legacy behaviour — returns a JSON array of - transactions (`limit`/`offset` window). - - With ``pagination`` JSON: returns ``{ items, pagination, groupLayout?, appliedView? }``. - Table list views use contextKey ``billing/transactions``. - """ - try: - billingService = getBillingService( - ctx.user, - ctx.mandateId, - featureCode="billing", - ) - - if pagination: - from modules.interfaces.interfaceTableHelpers import ( - applyViewToParams, - buildGroupLayout, - effective_group_by_levels, - resolveView, - ) - from modules.dbHelpers.paginationHelpers import ( - handleFilterValuesInMemory, - handleIdsInMemory, - ) - from modules.interfaces.interfaceDbApp import getInterface as getAppInterface - from modules.interfaces.interfaceDbManagement import ComponentObjects - - CONTEXT_KEY = "billing/transactions" - - try: - paginationDict = json.loads(pagination) - if not paginationDict: - raise ValueError("empty pagination") - paginationDict = normalize_pagination_dict(paginationDict) - paginationParams = PaginationParams(**paginationDict) - except (json.JSONDecodeError, ValueError, TypeError) as e: - raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") - - appInterface = getAppInterface(ctx.user) - viewKey = paginationParams.viewKey - viewConfig, viewDisplayName = resolveView(appInterface, CONTEXT_KEY, viewKey) - viewMeta = AppliedViewMeta(viewKey=viewKey, displayName=viewDisplayName) if viewKey else None - paginationParams = applyViewToParams(paginationParams, viewConfig) - groupByLevels = effective_group_by_levels(paginationParams, viewConfig) - - all_items = _load_billing_user_transactions_normalized(billingService) - - if mode == "filterValues": - if not column: - raise HTTPException(status_code=400, detail="column parameter required for mode=filterValues") - return handleFilterValuesInMemory(all_items, column, pagination) - - if mode == "ids": - return handleIdsInMemory(all_items, pagination) - - comp = ComponentObjects() - comp.setUserContext(ctx.user) - if paginationParams.filters: - all_items = comp._applyFilters(all_items, paginationParams.filters) - if paginationParams.sort: - all_items = comp._applySorting(all_items, paginationParams.sort) - - totalItems = len(all_items) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - - if not groupByLevels: - pstart = (paginationParams.page - 1) * paginationParams.pageSize - page_items = all_items[pstart : pstart + paginationParams.pageSize] - group_layout = None - else: - page_items, group_layout = buildGroupLayout( - all_items, - groupByLevels, - paginationParams.page, - paginationParams.pageSize, - ) - - resp: Dict[str, Any] = { - "items": page_items, - "pagination": PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort, - filters=paginationParams.filters, - ).model_dump(), - } - if group_layout: - resp["groupLayout"] = group_layout.model_dump() - if viewMeta: - resp["appliedView"] = viewMeta.model_dump() - return JSONResponse(content=resp) - - transactions = billingService.getTransactionHistory(limit=offset + limit) - result: List[TransactionResponse] = [] - for t in transactions[offset : offset + limit]: - result.append( - TransactionResponse( - id=t.get("id"), - accountId=t.get("accountId"), - transactionType=TransactionTypeEnum(t.get("transactionType", "DEBIT")), - amount=t.get("amount", 0.0), - description=t.get("description", ""), - referenceType=ReferenceTypeEnum(t["referenceType"]) if t.get("referenceType") else None, - workflowId=t.get("workflowId"), - featureCode=t.get("featureCode"), - featureInstanceId=t.get("featureInstanceId"), - aicoreProvider=t.get("aicoreProvider"), - aicoreModel=t.get("aicoreModel"), - createdByUserId=t.get("createdByUserId"), - sysCreatedAt=t.get("sysCreatedAt"), - mandateId=t.get("mandateId"), - mandateName=t.get("mandateName"), - ) - ) - return result - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting billing transactions: {e}") - raise HTTPException(status_code=500, detail=str(e)) - @router.get("/statistics", response_model=UsageReportResponse) @limiter.limit("30/minute") @@ -1373,51 +1215,6 @@ def getMandateViewTransactions( # User View Endpoints (RBAC-based) # ============================================================================= -@router.get("/view/users/balances", response_model=List[UserBalanceResponse]) -@limiter.limit("30/minute") -def getUserViewBalances( - request: Request, - ctx: RequestContext = Depends(getRequestContext) -): - """ - Get user-level balances. - - RBAC filtering: - - SysAdmin: sees all user balances across all mandates - - Mandate-Admin: sees user balances for mandates they administrate - - Regular user: sees only their own balances - """ - try: - billingInterface = getBillingInterface(ctx.user, ctx.mandateId) - - # Evaluate RBAC scope - scope = _getBillingDataScope(ctx.user) - - # Determine mandate IDs for data loading - if scope.isGlobalAdmin: - mandateIds = None - else: - mandateIds = scope.adminMandateIds + scope.memberMandateIds - if not mandateIds: - return [] - - allBalances = billingInterface.getUserBalancesForMandates(mandateIds) - - # RBAC filter: mandate admins see all in their mandates, regular users only own - if not scope.isGlobalAdmin: - adminMandateSet = set(scope.adminMandateIds) - allBalances = [ - b for b in allBalances - if b.get("mandateId") in adminMandateSet or b.get("userId") == scope.userId - ] - - return [UserBalanceResponse(**b) for b in allBalances] - - except Exception as e: - logger.error(f"Error getting user view balances: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - class ViewStatisticsResponse(BaseModel): """Aggregated statistics across all user's mandates.""" totalCost: float = 0.0 @@ -1730,25 +1527,48 @@ def getUserViewTransactions( resp["appliedView"] = viewMeta.model_dump(mode="json") return JSONResponse(content=resp) - result = billingInterface.getTransactionsForMandatesPaginated( - mandateIds=loadMandateIds, - pagination=paginationParams, - scope=effectiveScope, - userId=personalUserId, + _ENRICHED_FILTER_COLS = {"mandateName", "userName", "mandateId", "userId"} + _hasEnrichedFilters = paginationParams.filters and any( + k in _ENRICHED_FILTER_COLS for k in paginationParams.filters ) + if _hasEnrichedFilters: + all_items = _view_user_transactions_filtered_list( + billingInterface, + loadMandateIds, + effectiveScope, + personalUserId, + paginationParams, + ctx.user, + ) + totalItems = len(all_items) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + pstart = (paginationParams.page - 1) * paginationParams.pageSize + page_items = all_items[pstart : pstart + paginationParams.pageSize] + else: + result = billingInterface.getTransactionsForMandatesPaginated( + mandateIds=loadMandateIds, + pagination=paginationParams, + scope=effectiveScope, + userId=personalUserId, + ) + page_items = result.items + totalItems = result.totalItems + totalPages = result.totalPages + logger.debug( - f"SQL-paginated {result.totalItems} transactions for user {ctx.user.id} " - f"(scope={scope}, mandateId={mandateId}, page={paginationParams.page})" + f"Paginated {totalItems} transactions for user {ctx.user.id} " + f"(scope={scope}, mandateId={mandateId}, page={paginationParams.page}, " + f"enrichedFilter={_hasEnrichedFilters})" ) return PaginatedResponse( - items=[_toResponse(d) for d in result.items], + items=[_toResponse(d) for d in page_items], pagination=PaginationMetadata( currentPage=paginationParams.page, pageSize=paginationParams.pageSize, - totalItems=result.totalItems, - totalPages=result.totalPages, + totalItems=totalItems, + totalPages=totalPages, sort=paginationParams.sort, filters=paginationParams.filters, ), diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 745027bc..2d20f5c2 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -311,72 +311,6 @@ def get_folder_tree( raise HTTPException(status_code=500, detail=str(e)) -@router.post("/attributes") -@limiter.limit("120/minute") -def getAttributesForIds( - request: Request, - body: Dict[str, Any] = Body(...), - currentUser: User = Depends(getCurrentUser), - context: RequestContext = Depends(getRequestContext), -): - """Return current attribute values (neutralize, scope, ragIndexEnabled) for - a list of node IDs. For folder IDs, computes 'mixed' by checking direct - children. The frontend sends this after every toggle to refresh visible - nodes without reloading the tree structure.""" - ids = body.get("ids", []) - if not isinstance(ids, list) or len(ids) == 0: - return {} - if len(ids) > 500: - raise HTTPException(status_code=400, detail="Max 500 IDs per request") - - try: - managementInterface = interfaceDbManagement.getInterface( - currentUser, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, - ) - db = managementInterface.db - userId = str(currentUser.id) - - allFolders = db.getRecordset(FileFolder, recordFilter={"sysCreatedBy": userId}) or [] - allFiles = db.getRecordset(FileItem, recordFilter={"sysCreatedBy": userId}) or [] - - folderById = {f["id"]: f for f in allFolders} - fileById = {f["id"]: f for f in allFiles} - - logger.info( - "getAttributesForIds: %d ids requested, %d folders found, %d files found", - len(ids), len(allFolders), len(allFiles), - ) - - result: Dict[str, Dict[str, Any]] = {} - - for nodeId in ids: - if nodeId.startswith("__filesRoot:"): - attrs = _computeSyntheticRootAttrs(allFolders, allFiles) - result[nodeId] = attrs - elif nodeId in folderById: - folder = folderById[nodeId] - attrs = _computeFolderAttrs(folder, allFolders, allFiles) - result[nodeId] = attrs - elif nodeId in fileById: - f = fileById[nodeId] - result[nodeId] = { - "neutralize": bool(f.get("neutralize", False)), - "scope": f.get("scope", "personal"), - } - else: - logger.debug("getAttributesForIds: unknown id=%s", nodeId) - - logger.info("getAttributesForIds: returning %d entries", len(result)) - return result - except HTTPException: - raise - except Exception as e: - logger.error(f"getAttributesForIds error: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - def _enrichFoldersWithMixed( db, userId: str, folders: List[Dict[str, Any]], ownerMode: str, ) -> None: @@ -480,43 +414,6 @@ def _effectiveScope( return childVals.pop() -def _computeSyntheticRootAttrs( - allFolders: List[Dict[str, Any]], - allFiles: List[Dict[str, Any]], -) -> Dict[str, Any]: - """Compute attributes for the synthetic root by recursively checking the - entire tree. If ANY item at any depth diverges, root shows 'mixed'.""" - topFolders = [f for f in allFolders if not f.get("parentId")] - topFiles = [f for f in allFiles if not f.get("folderId")] - - neutralizeVals = set() - scopeVals = set() - for cf in topFolders: - nEff = _effectiveNeutralize(cf["id"], allFolders, allFiles) - if nEff == "mixed": - neutralizeVals.add(True) - neutralizeVals.add(False) - else: - neutralizeVals.add(nEff) - sEff = _effectiveScope(cf["id"], allFolders, allFiles) - if sEff == "mixed": - scopeVals.add("__mixed_a__") - scopeVals.add("__mixed_b__") - else: - scopeVals.add(sEff) - for cf in topFiles: - neutralizeVals.add(bool(cf.get("neutralize", False))) - scopeVals.add(cf.get("scope", "personal")) - - if not neutralizeVals and not scopeVals: - return {"neutralize": False, "scope": "personal"} - - return { - "neutralize": "mixed" if len(neutralizeVals) > 1 else (neutralizeVals.pop() if neutralizeVals else False), - "scope": "mixed" if len(scopeVals) > 1 else (scopeVals.pop() if scopeVals else "personal"), - } - - @router.post("/folders", status_code=status.HTTP_201_CREATED) @limiter.limit("30/minute") def create_folder( @@ -1133,132 +1030,6 @@ def batchDownload( raise HTTPException(status_code=500, detail=str(e)) -# ── Bulk file operations (replace former group-based bulk routes) ───────────── - -@router.post("/bulk/scope") -@limiter.limit("30/minute") -def bulk_set_scope( - request: Request, - body: dict = Body(...), - currentUser: User = Depends(getCurrentUser), - context: RequestContext = Depends(getRequestContext), -): - """Set scope for a list of files by their IDs.""" - fileIds: list = body.get("fileIds") or [] - scope: str = body.get("scope") or "" - if not fileIds: - raise HTTPException(status_code=400, detail="fileIds is required") - validScopes = {"personal", "featureInstance", "mandate", "global"} - if scope not in validScopes: - raise HTTPException(status_code=400, detail=f"Invalid scope. Must be one of {validScopes}") - if scope == "global" and not context.isSysAdmin: - raise HTTPException(status_code=403, detail="Only sysadmins can set global scope") - try: - managementInterface = interfaceDbManagement.getInterface( - currentUser, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, - ) - updated = 0 - for fid in fileIds: - try: - managementInterface.updateFile(fid, {"scope": scope}) - updated += 1 - except Exception as e: - logger.error(f"bulk_set_scope: failed for file {fid}: {e}") - return {"scope": scope, "filesUpdated": updated} - except HTTPException: - raise - except Exception as e: - logger.error(f"bulk_set_scope error: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/bulk/neutralize") -@limiter.limit("30/minute") -def bulk_set_neutralize( - request: Request, - body: dict = Body(...), - currentUser: User = Depends(getCurrentUser), - context: RequestContext = Depends(getRequestContext), -): - """Set neutralize flag for a list of files by their IDs (incl. knowledge purge/reindex).""" - fileIds: list = body.get("fileIds") or [] - neutralize = body.get("neutralize") - if not fileIds: - raise HTTPException(status_code=400, detail="fileIds is required") - if neutralize is None: - raise HTTPException(status_code=400, detail="neutralize is required") - try: - managementInterface = interfaceDbManagement.getInterface( - currentUser, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, - ) - updated = 0 - for fid in fileIds: - try: - managementInterface.updateFile(fid, {"neutralize": neutralize}) - if not neutralize: - try: - kIface = interfaceDbKnowledge.getInterface(currentUser) - kIface.purgeFileKnowledge(fid) - except Exception as ke: - logger.warning(f"bulk_set_neutralize: knowledge purge failed for {fid}: {ke}") - updated += 1 - except Exception as e: - logger.error(f"bulk_set_neutralize: failed for file {fid}: {e}") - return {"neutralize": neutralize, "filesUpdated": updated} - except HTTPException: - raise - except Exception as e: - logger.error(f"bulk_set_neutralize error: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/bulk/download-zip") -@limiter.limit("10/minute") -async def bulk_download_zip( - request: Request, - body: dict = Body(...), - currentUser: User = Depends(getCurrentUser), - context: RequestContext = Depends(getRequestContext), -): - """Download a list of files as a ZIP archive.""" - fileIds: list = body.get("fileIds") or [] - if not fileIds: - raise HTTPException(status_code=400, detail="fileIds is required") - try: - managementInterface = interfaceDbManagement.getInterface( - currentUser, - mandateId=str(context.mandateId) if context.mandateId else None, - featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None, - ) - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for fid in fileIds: - try: - fileMeta = managementInterface.getFile(fid) - fileData = managementInterface.getFileData(fid) - if fileMeta and fileData: - name = (getattr(fileMeta, "fileName", None) or fid) - zf.writestr(name, fileData) - except Exception as fe: - logger.warning(f"bulk_download_zip: skipping file {fid}: {fe}") - buf.seek(0) - from fastapi.responses import StreamingResponse - return StreamingResponse( - buf, - media_type="application/zip", - headers={"Content-Disposition": 'attachment; filename="files.zip"'}, - ) - except HTTPException: - raise - except Exception as e: - logger.error(f"bulk_download_zip error: {e}") - raise HTTPException(status_code=500, detail=str(e)) - - # ── Scope & neutralize tagging endpoints (before /{fileId} catch-all) ───────── @router.patch("/{fileId}/scope") diff --git a/modules/routes/routeJobs.py b/modules/routes/routeJobs.py index c98c7bd0..7ef8c52b 100644 --- a/modules/routes/routeJobs.py +++ b/modules/routes/routeJobs.py @@ -4,7 +4,6 @@ Endpoints: - GET /api/jobs/{jobId} -> single job status -- GET /api/jobs -> list (filter by jobType, instanceId) Access control: a caller may read a job iff they are a member of its mandate (or PlatformAdmin). Jobs without a mandateId (system-wide) are restricted to @@ -12,14 +11,13 @@ PlatformAdmin only. """ import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional -from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Path, Request from modules.auth import getRequestContext, RequestContext, limiter from modules.serviceCenter.services.serviceBackgroundJobs import ( getJobStatus, - listJobs, ) from modules.shared.i18nRegistry import apiRouteContext, resolveJobMessage @@ -91,29 +89,3 @@ def get_job( return _serialiseJob(job) -@router.get("") -@limiter.limit("30/minute") -def list_jobs( - request: Request, - jobType: Optional[str] = Query(None), - mandateId: Optional[str] = Query(None), - instanceId: Optional[str] = Query(None, description="Feature instance scope"), - limit: int = Query(20, ge=1, le=100), - context: RequestContext = Depends(getRequestContext), -) -> Dict[str, List[Dict[str, Any]]]: - """List recent jobs filtered by scope. Newest first.""" - if mandateId is None: - if not context or not context.isPlatformAdmin: - raise HTTPException( - status_code=400, - detail=routeApiMsg("mandateId is required (only PlatformAdmin may list system-wide)"), - ) - elif not _userHasMandateAccess(context, mandateId): - raise HTTPException(status_code=403, detail=routeApiMsg("Access denied")) - jobs = listJobs( - mandateId=mandateId, - featureInstanceId=instanceId, - jobType=jobType, - limit=limit, - ) - return {"items": [_serialiseJob(j) for j in jobs]} diff --git a/modules/routes/routeVoiceGoogle.py b/modules/routes/routeVoiceGoogle.py index ada38504..addf7e47 100644 --- a/modules/routes/routeVoiceGoogle.py +++ b/modules/routes/routeVoiceGoogle.py @@ -14,10 +14,8 @@ import secrets import time from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect from typing import Optional, Dict, Any, List -from modules.auth import getCurrentUser, getRequestContext, RequestContext, limiter -from modules.datamodels.datamodelUam import User +from modules.auth import getRequestContext, RequestContext, limiter from modules.interfaces.interfaceVoiceObjects import getVoiceInterface, VoiceObjects -from modules.shared.voiceCatalog import getCatalogPayload logger = logging.getLogger(__name__) router = APIRouter(prefix="/voice-google", tags=["Voice Google"]) @@ -49,70 +47,6 @@ class ConnectionManager: manager = ConnectionManager() -def _getVoiceInterface(currentUser: User) -> VoiceObjects: - """Get voice interface instance with user context.""" - try: - return getVoiceInterface(currentUser) - except Exception as e: - logger.error(f"Failed to initialize voice interface: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to initialize voice interface: {str(e)}" - ) - -@router.get("/languages") -async def get_available_languages(currentUser: User = Depends(getCurrentUser)): - """Return the curated voice/language catalog (single source of truth). - - Each entry: {bcp47, iso, label, flag, defaultVoice}. Same payload as - /api/voice/languages — both endpoints back the same catalog. - """ - return { - "success": True, - "languages": getCatalogPayload(), - } - -@router.get("/voices") -async def get_available_voices( - languageCode: Optional[str] = None, - language_code: Optional[str] = None, # Accept both camelCase and snake_case - currentUser: User = Depends(getCurrentUser) -): - """ - Get available voices from Google Cloud Text-to-Speech. - Accepts languageCode (camelCase) or language_code (snake_case) query parameter. - """ - # Use language_code if provided (frontend sends this), otherwise use languageCode - if language_code: - languageCode = language_code - - try: - logger.info(f"🎤 Getting available voices, language filter: {languageCode}") - - voiceInterface = _getVoiceInterface(currentUser) - result = await voiceInterface.getAvailableVoices(languageCode=languageCode) - - if result["success"]: - return { - "success": True, - "voices": result["voices"], - "language_filter": languageCode - } - else: - raise HTTPException( - status_code=400, - detail=f"Failed to get voices: {result.get('error', 'Unknown error')}" - ) - - except HTTPException: - raise - except Exception as e: - logger.error(f"❌ Get voices error: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to get available voices: {str(e)}" - ) - # ========================================================================= # STT Streaming WebSocket — generic, used by all features # ========================================================================= diff --git a/modules/routes/routeWorkflowAutomation.py b/modules/routes/routeWorkflowAutomation.py index 619928f9..09dc311d 100644 --- a/modules/routes/routeWorkflowAutomation.py +++ b/modules/routes/routeWorkflowAutomation.py @@ -207,27 +207,6 @@ async def _listRuns( db.close() -@router.get("/runs/{runId}") -async def _getRun( - runId: str, - request: RequestContext = Depends(getRequestContext), -): - db = _getWorkflowAutomationDb() - try: - db._ensureTableExists(AutoRun) - run = db.getRecord(AutoRun, runId) - if not run: - raise HTTPException(status_code=404, detail="Run not found") - - wfId = run.get("workflowId") - if wfId: - wf = db.getRecord(AutoWorkflow, wfId) - _validateWorkflowAccess(request, wf, "read") - return run - finally: - db.close() - - # --------------------------------------------------------------------------- # Tasks # --------------------------------------------------------------------------- From 04381779342d5e392194e1c13c2cf455b84650c6 Mon Sep 17 00:00:00 2001 From: Ida Date: Fri, 12 Jun 2026 09:28:11 +0200 Subject: [PATCH 7/8] =?UTF-8?q?FIX:=20mandant=20wird=20erstellt=20wenn=20s?= =?UTF-8?q?ich=20ein=20user=20mit=20microsoft=20oder=20google=20das=20erst?= =?UTF-8?q?e=20mal=20registriert,=20damit=20er=20die=20app=20tats=C3=A4chl?= =?UTF-8?q?ich=20nutzen=20kann?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/auth/homeMandateService.py | 55 +++++++++++++++++++++++++++ modules/routes/routeMfa.py | 15 ++++++++ modules/routes/routeSecurityGoogle.py | 6 +++ modules/routes/routeSecurityLocal.py | 48 ++--------------------- modules/routes/routeSecurityMsft.py | 15 ++++++++ 5 files changed, 94 insertions(+), 45 deletions(-) create mode 100644 modules/auth/homeMandateService.py diff --git a/modules/auth/homeMandateService.py b/modules/auth/homeMandateService.py new file mode 100644 index 00000000..97e70683 --- /dev/null +++ b/modules/auth/homeMandateService.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026 PowerOn AG +# All rights reserved. +"""Ensure new users receive a Home mandate on first login.""" + +import logging + +logger = logging.getLogger(__name__) + + +def ensureHomeMandate(rootInterface, user) -> None: + """Ensure user has a Home mandate, but only if they have no mandate memberships + AND no pending invitations. + + Invited users should NOT get a Home mandate — they join existing mandates via + invitation acceptance and can create their own later via onboarding. + """ + userId = str(user.id) + userMandates = rootInterface.getUserMandates(userId) + + if userMandates: + for um in userMandates: + mandate = rootInterface.getMandate(um.mandateId) + if mandate and (mandate.name or "").startswith("Home ") and not mandate.isSystem: + return + logger.debug( + f"User {user.username} has {len(userMandates)} mandate(s) but no Home — skipping auto-creation" + ) + return + + try: + normalizedEmail = (user.email or "").strip().lower() if user.email else None + pendingByUsername = rootInterface.getInvitationsByTargetUsername(user.username) + pendingByEmail = ( + rootInterface.getInvitationsByEmail(normalizedEmail) if normalizedEmail else [] + ) + seenIds = set() + for inv in pendingByUsername + pendingByEmail: + if inv.id in seenIds: + continue + seenIds.add(inv.id) + if not inv.revokedAt and (inv.currentUses or 0) < (inv.maxUses or 1): + logger.info( + f"User {user.username} has pending invitation(s) — skipping Home mandate creation" + ) + return + except Exception as e: + logger.warning(f"Could not check pending invitations for {user.username}: {e}") + + homeMandateLabel = f"Home {user.username}" + rootInterface._provisionMandateForUser( + userId=userId, + mandateLabel=homeMandateLabel, + planKey="TRIAL_14D", + ) + logger.info(f"Created Home mandate '{homeMandateLabel}' for user {user.username}") diff --git a/modules/routes/routeMfa.py b/modules/routes/routeMfa.py index 6dafdf4e..eb66965b 100644 --- a/modules/routes/routeMfa.py +++ b/modules/routes/routeMfa.py @@ -26,6 +26,7 @@ from modules.auth import ( setAccessTokenCookie, setRefreshTokenCookie, ) +from modules.auth.homeMandateService import ensureHomeMandate from modules.auth.mfaService import ( generateSetup, confirmSetup, @@ -229,6 +230,20 @@ def mfaVerify( ) userInterface.saveAccessToken(dbToken) + try: + ensureHomeMandate(rootInterface, user) + except Exception as homeErr: + logger.error(f"Error ensuring Home mandate for user {username}: {homeErr}") + + try: + activatedCount = rootInterface._activatePendingSubscriptions(userId) + if activatedCount > 0: + logger.info( + f"Activated {activatedCount} pending subscription(s) for user {username} after MFA" + ) + except Exception as subErr: + logger.error(f"Error activating subscriptions after MFA verify: {subErr}") + logger.info("MFA verify successful for user %s", username) # Mark device as trusted so MFA is skipped on next login from this device diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index a363f7bf..d94876bd 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -34,6 +34,7 @@ from modules.auth import ( ) from modules.auth.tokenManager import TokenManager from modules.auth.oauthProviderConfig import googleAuthScopes, googleDataScopes +from modules.auth.homeMandateService import ensureHomeMandate from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeSecurityGoogle") @@ -278,6 +279,11 @@ async def auth_login_callback( ) # --- end MFA gate ----------------------------------------------------- + try: + ensureHomeMandate(rootInterface, user) + except Exception as homeErr: + logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}") + jwt_token_data = { "sub": user.username, "userId": str(user.id), diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 2b873e26..04a2c666 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -16,12 +16,13 @@ from jose import jwt # Import auth modules from modules.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM, getRequestContext, RequestContext from modules.auth import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie, clearAccessTokenCookie, clearRefreshTokenCookie -from modules.interfaces.interfaceDbApp import getInterface, getRootInterface, getRootInterface as _getRootIf +from modules.interfaces.interfaceDbApp import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate from modules.datamodels.datamodelSecurity import Token, TokenPurpose from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcTimestamp from modules.shared.i18nRegistry import apiRouteContext +from modules.auth.homeMandateService import ensureHomeMandate routeApiMsg = apiRouteContext("routeSecurityLocal") # Configure logger @@ -174,49 +175,6 @@ router = APIRouter( } ) -def _ensureHomeMandate(rootInterface, user) -> None: - """Ensure user has a Home mandate, but only if they have no mandate memberships - AND no pending invitations. - - Invited users should NOT get a Home mandate — they join existing mandates via - invitation acceptance and can create their own later via onboarding. - """ - userId = str(user.id) - userMandates = rootInterface.getUserMandates(userId) - - if userMandates: - for um in userMandates: - mandate = rootInterface.getMandate(um.mandateId) - if mandate and (mandate.name or "").startswith("Home ") and not mandate.isSystem: - return - logger.debug(f"User {user.username} has {len(userMandates)} mandate(s) but no Home — skipping auto-creation") - return - - try: - appIf = _getRootIf() - normalizedEmail = (user.email or "").strip().lower() if user.email else None - pendingByUsername = appIf.getInvitationsByTargetUsername(user.username) - pendingByEmail = appIf.getInvitationsByEmail(normalizedEmail) if normalizedEmail else [] - seenIds = set() - for inv in pendingByUsername + pendingByEmail: - if inv.id in seenIds: - continue - seenIds.add(inv.id) - if not inv.revokedAt and (inv.currentUses or 0) < (inv.maxUses or 1): - logger.info(f"User {user.username} has pending invitation(s) — skipping Home mandate creation") - return - except Exception as e: - logger.warning(f"Could not check pending invitations for {user.username}: {e}") - - homeMandateLabel = f"Home {user.username}" - rootInterface._provisionMandateForUser( - userId=userId, - mandateLabel=homeMandateLabel, - planKey="TRIAL_14D", - ) - logger.info(f"Created Home mandate '{homeMandateLabel}' for user {user.username}") - - @router.post("/login") @limiter.limit("30/minute") def login( @@ -364,7 +322,7 @@ def login( # Ensure user has a Home mandate (created on first login if missing) try: - _ensureHomeMandate(rootInterface, user) + ensureHomeMandate(rootInterface, user) except Exception as homeErr: logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}") diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 45d3deda..6325cfa8 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -35,6 +35,7 @@ from modules.auth import ( ) from modules.auth.tokenManager import TokenManager from modules.auth.oauthProviderConfig import msftAuthScopes, msftDataScopes, msftDataScopesForRefresh +from modules.auth.homeMandateService import ensureHomeMandate from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeSecurityMsft") @@ -251,6 +252,20 @@ async def auth_login_callback( ) # --- end MFA gate ----------------------------------------------------- + try: + ensureHomeMandate(rootInterface, user) + except Exception as homeErr: + logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}") + + try: + activatedCount = rootInterface._activatePendingSubscriptions(str(user.id)) + if activatedCount > 0: + logger.info( + f"Activated {activatedCount} pending subscription(s) for user {user.username}" + ) + except Exception as subErr: + logger.error(f"Error activating subscriptions on Microsoft login: {subErr}") + jwt_token_data = { "sub": user.username, "userId": str(user.id), From f8976d1d9584e45fa90b9833aa64efd734407900 Mon Sep 17 00:00:00 2001 From: Ida Date: Fri, 12 Jun 2026 10:21:52 +0200 Subject: [PATCH 8/8] =?UTF-8?q?FIX:=20Home-Mandat=20bei=20OAuth-Login:=20N?= =?UTF-8?q?eue=20MSFT/Google-User=20(und=20nach=20MFA)=20bekommen=20automa?= =?UTF-8?q?tisch=20ein=20Home-Mandat=20mit=20TRIAL=5F14D,=20damit=20Store,?= =?UTF-8?q?=20Verbindungen=20und=20Dateien=20funktionieren.=20Microsoft=20?= =?UTF-8?q?Scopes=20=E2=86=92=20.default:=20Login-=20und=20Connect-Flow=20?= =?UTF-8?q?nutzen=20https://graph.microsoft.com/.default=20statt=20Einzel-?= =?UTF-8?q?Scopes,=20damit=20tenant-weiter=20Admin-Consent=20greift.=20Aut?= =?UTF-8?q?hority=20/organizations:=20Service=5FMSFT=5FTENANT=5FID=20in=20?= =?UTF-8?q?allen=20env-Dateien=20von=20common=20auf=20organizations=20(nur?= =?UTF-8?q?=20Gesch=C3=A4ftskonten).=20Admin-Consent-Callback:=20Fehlendes?= =?UTF-8?q?=20state=20f=C3=BChrt=20nicht=20mehr=20zu=20hartem=20Fehler,=20?= =?UTF-8?q?wenn=20Consent=20au=C3=9Ferhalb=20unserer=20Route=20gestartet?= =?UTF-8?q?=20wurde.=20Token-Refresh=20geh=C3=A4rtet:=20Proaktives=20Refre?= =?UTF-8?q?sh-Fenster=20von=205=20auf=2030=20Minuten,=20Rate-Limit=20von?= =?UTF-8?q?=203=20auf=206=20pro=20Stunde.=20Connect-Refresh=20ohne=20promp?= =?UTF-8?q?t=3Dconsent:=20Beim=20=E2=80=9EVerbindung=20aktualisieren?= =?UTF-8?q?=E2=80=9C=20(reauth=3D1)=20nur=20noch=20select=5Faccount,=20kei?= =?UTF-8?q?n=20erzwungener=20Consent=20=E2=86=92=20kein=20=E2=80=9ENeed=20?= =?UTF-8?q?admin=20approval=E2=80=9C=20f=C3=BCr=20normale=20User.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- env-dev.env | 2 +- env-int.env | 2 +- env-prod.env | 2 +- modules/auth/oauthProviderConfig.py | 15 +++++++ modules/auth/tokenRefreshService.py | 19 ++++++--- modules/routes/routeSecurityMsft.py | 61 ++++++++++++++++------------- 6 files changed, 64 insertions(+), 37 deletions(-) diff --git a/env-dev.env b/env-dev.env index 179f7caf..5e70dcb0 100644 --- a/env-dev.env +++ b/env-dev.env @@ -71,7 +71,7 @@ Connector_AiTavily_API_SECRET = DEV_ENC:Z0FBQUFBQnFCdlFlY1R2WGpuazk5M05SeDIyLWd3 Connector_AiPrivateLlm_API_SECRET = DEV_ENC:Z0FBQUFBQnBudkpGRHM5eFdUVmVZU1R1cHBwN1RlMUx4T0NlLTJLUFFVX3J2OElDWFpuZmJHVmp4Z3BNNWMwZUVVZUd2TFhRSjVmVkVlcFlVRWtybXh0ZHloZ01ZcnVvX195YjdlWVdEcjZSWFFTTlNBWUlaTlNoLWhqVFBIb0thVlBiaWhjYjFQOFY= Connector_AiMistral_API_SECRET = DEV_ENC:Z0FBQUFBQnFCdlFlelh2T2hqNGcxV0hMV1FKbmFDZjVHUWF6T2FXbGlCSnQzSzNXLWJHeXBFWE1nUlh1b1NHY1JRSEVtTVEtc1MtUnZrX2ZCcURqQ2FYNmFWa2xudGJtS3g2eVo4MFZMd09nZTBNMmo1ZHU0bzBJdFRqLVhHSVZNb2Zrc0VkUXI0SVk= -Service_MSFT_TENANT_ID = common +Service_MSFT_TENANT_ID = organizations # Google Cloud Speech Services configuration Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQnFIc3YtU0x4LTlHbTY1NUVGY2V2bUdmck85dDh1ZWVKa2ktR0N6NjdlTGFrUHMybVQ2bVRLN01XNFRZR2lyN0ZNSHhzWVVGNnVtZjRjV2hhR0ViTDYwT25lSmxJY0pSTkl3OUEyT0JxMFVYRndfUFJudExMajdTYUNXS01JU2lhQzZmNWFYdXA4aVZ5Zkh4Zko1Z00tcEE5ZFEwQkFVa1oyR296YXozRFI2WUdXN0ZSREFFclFNaTd6OUVlSmFxS1BTSlNJbnlWNHNfbkk4QzVOUGlkMzdfQUZxUlJOVEZzUlN1aWRWY01JZmlRM0JNZE1EZ3BmbW10c3BDdERpa2FMakstQUlqVEVlRC1hUmZoeFVoQ3pYNXRlRFVSTlI3ekJrU0QwSHBSaWxiSGU0akFGMXUtY2Q0RnUzS0tPOEQtcTdVdWhQeHFDM1hRRVVMcUxCeklvWHNWRUN2bjVHZUUwLTVtaGpUbWdPUnJabWlIcHZ5UjNtN0NMTUNRN29ZRGVXU28xQmhJTVg2eEZnaUdrcW9UVklHMHJycm1nT0JkdGJReVVHeV8tYm12UDlOU0lpNHFidXBQbUFSSVVmWUl1M1BVMFFncm0xSldkVzBrb2poRFMyaVUwcUZvMHl0QlZIZ1h1MjZwR3AtZWhqdzN4UVhtT2hUa1lQU3VudzNXdW1FcVY3VnQ3RmpkQnFQemlrQlF3WGhBNWxOZXJ6Zm9KVFlEZExUXzlqODhYaFNNMzVWTzFNMmVTcWdodDZoRmZTUzlhLVlOSU5fYW1vNXctaFpFMC1pUllRZW11d1JQN25sbldHVjI1anc2UC1ycndjTGtxWk55WmpJeU1wOVR0RnlTdFpad1dkRmlUNDE0d240TDlKc3JFUXdOYzd5UTFYSXUzLTQ2Y1ZGcWE3R2RyQ0I1WDMtMHBScEFzZDV4UEkyanh4ckJZUjdTYnJGZjAxQkU3MEJ6OXdybGRaWHNod1hZZEhVOXRpMWRLbVJsRGd0UDRDN3JsRzF4T0RpcnczRU5TM0RKVjVkWTRqNTl6bmhQdmdvaEg1U2kya0QtQ0l4ZHVUcGxkNi1vNVVVOEcyWXhxZWc5N1lKMk4tT0o3ZFVzYjJtT3NVZFJiSTFNUnpaSmFOeDZaLWVpZlc0VUhZRHdXOUMyQ3cwaXBQUDRJN1g1YkwzaTFiRVRxRFY5UTdZU1dSaGR6NUw3aEtac2RENXF3WEpVN0dXVTlQR0F6MFlpWl83MU44NVR1ZUtPVUNlZ205YUIwOFoxUDBvTlI0SU52emVvQ3VZXy1jTlFXRWZXQ0d5RHJ0eV9JeE5wMHl0b3FVSjNoVzg2d21hYVNYY3Q0dkFaVEZwa09tRnFBbEtoOUlGY2xkeVJoZGYzQUxYNFZfb0ZiaU5VRjJPbGhieXYtWTFKckZwenVCUGFva1IwVVFORVQ4SDMxWHVuRWhBRGd0cVlsc3kyQ0RyY2ZIVDlwcGh5ampySV9uOVpsVmlWbGoxMEg3SXh6NzRJbmZXRlhMMWc0RXhzeWtnQlJ0VnZSdENkbEpOdENwUzItUjZhZWFYRFhzbDM1WDBxaGFPX19CSG1KZjRTTU5JemcxZzJRSFY5bkx4TTlIZFNHOW1USWxBYWhEZ1FSNVdSSDJETUZwMi1Hd0RESkF2cVA1TVJGTEtPUl9oN3gzVEIwSzZOVzlOWXhNa2I1Vzc1SV9tdENfRy1rQTNzRlZGSTYwQmJIaGswZUNWSnRDVXFfdWFCckZZcnJOT2Rfb3FrcWI4S1lVRTMyRnZJQTRZV1VsU0xobGRjekhtbG9LamR2d1hfVklsM3JBeW9SRzJnWVdiWDRzN1ltcXdSVGoxRVBvczViVXNjMUxBazZUdS1WbkRQX0h1MzdNd3ltVDUzd2FGdi1XeUMybV9ia1YxQVBPdnUxY1dfT2M5eEpZR2JHMkdZbWdDZTRERXRYOWxodndkTXltVW40c0t0bVA5YWxuRzM3LWlCdmJiYmF5dkNBY3ozbUw1Zm5zRmpBdk5ORmFZRWJKM3Q2UDdKNl9zaUV5eVVGbkF0QmZSZzk5dGo3UjNIQWxwcjRlVTdUT2s1VGFjdndvX2c3d1VmaHRMZU10M1ZKVk9Ma3dZb1kwYVV5Z2NlTjUxdUYtZXRnRTRzQlp1aFp0OUF5TVBwN1gzU21kRmJ6OUlOeUFOOEhEOU5WSENNZndvLXdoVUFJYVFDTWEyakJEcTVSVDhJOWJscU8taThqNUZkdThCOUlXcldndFBTZk9QVnlMaUphUU5sUktpb1plZDZOQnFzNFNMUzRWbWFVQWhUWmJfem96X0cxWXVTcUxCeDhOc3E2OEpFa2lzWHFIV0p3eGdBZmN1aXBhYjExZTZqaUY4S0ZudTNhcUx2WlpuTU9lNUk2ZmNyN0JCODdYMGNEU2JsZkZXYlRFaTJQUTI5RU5SMmtkV1NHQTVTTjEyZGZLYnhTNTg2Nl9aaWJqX2Q1U1NwQ3pRTGRBSUw0N3FNQ0ItMks1QVZmbURYVWdHMWFZTWhGNURVOUg0bGVuMUozanlxTnRwbVlGX2RnN2FBVTZlZjhDaXVzZEtVR1Z5azhzWHRrS1dYSG9rYkowTjQ1N0hyRWdNVWMya1ZmWmZvSnVTdHNiMHFDODNLckpjQ081SFlieGxuM0picGhKMnNQRURwY2hpQzF3dHRnNEFWcUlPYjVxZEhod0JDbWZhU01Ob21UWmRwd0NQRlpjOE5CUFBOT004U2JKNkFSUlFzRklYZGJobUoxQzZzT2wzZ3J1Z05aYThRVVNzcFktMGJDcXFfSkxVS2hhajI3dTdrR2poa21ZM3Z4UzFRblFsOFlOZVVUM0YxaFRuNjFWQ2E4ZlhvZjZpMWFtOGRuaGx0MTZxZE9TY1dsTTMyMHhsNXJ2MkduaGRkZXpYUWJ3cEt1U3YwMC1IRzM5eWRCb0lvaUhTQ2R4XzhEZl9zRk5GeHhCSWx2X3BkUkJ4NFZLVzdVRFZkbnpNNkpjUTFHY1pDV0ZOMFBaNTVpLUlmSnFrX1N5X05MTjRUeTVERUs5MG9kMFJ3di03U3BpMUM4YXNwaG1fangwYURIVjBpSVdCUkt4UW5HbWtGOUh3TUdPZjMxYXpVZDcwTmlDcTR6WldZb3VzbHRpRUgyN2lFTjlpUV85T0M4blJxMWx0cC1iU0FDOHhueDBLYjdLZGhNbjFPbE1RdmhhNlEzX3ZpT2ZsYllwNkU5TE9fZWFabDE4RWRoRWxiMk5aVFZrWmxjaW5MX1VrUGhUN29vbU1tWldESnczYTNBQ1RPd1VTNGNJdjdJU3p3QXZQLVlDNkQ1cTh4Rk1WNnRMUi1DT3VGREFPa28xejc2NUl1dzJSa2hCTlJublBRNGkydlJVRjlFbFotOWtraWFqQkNNTXBpT1hZM0NXNEpObGMxQUNuS29rOExMSnMxT3NLbjNfLTdpQW1BcDMxR1RZdVRvbElGbENWbHJqRlVrTXhYbFdiMmItUzlxR2ZxT2FCWXpMVVJYZXBfSFVwNTczU3JHUVhET3hSWm80Ry1KcE9mV3FYejVHSEVSS0pxOUtCc3V2VHNFVkRqYk5Od20tM0ttdFQ1eGdsc091WGFYNFgybzNVd3ZvbzEwUDJ0T0hvTVd3YnlHNnpNWC0wbkJOQTIwQ3VYdlUzaXY5NFhDNlNOOW9UdGZNUk4zZ0VJakpwS21SZlJtQjVWLUxfejFYZFc1cjRwR3ZUOGdZb2VJaTdJUS1MYlRJb0ZFYW9uYzM3MDd4b09BR1pnTEh3RFpnaGhxZURQamllNUhqTHg0cHJfN08wMkdGSVQwQUlqWDhLVGViY3J5NlVFTzY3RGhGQ0R6aXNsb2w4dnBVYndTd1Jhd3IwS1BxY0h1X05RcGsySzVNbXR5YlBVQi1IOGFUNkh5QjhRZk5BQmZvcGF6ZTNXenZkdy1GRjFGdE1saGdMSnotUkIyX1VqTlZFWnJER1YyNGQtMFZHU3hmRVNPUWFCdXV3QUxzOGVSbF9EdEZGUFNxbTdiYm5oWHdYak5qa3Zoem5WY1ZUdDREVUxGX0VQeS1jckhqS2lRLXQ1Y2tyOFRjYnVhajNUZmZOUE9kbU9PYXdqdk5DYUtEOVFiMW9yZTYxMFNUaDdvUTExUFZ1bklYSkRKTnJ1RURvOTR3ODREcWdWeHpRS2RETjZqeXpvbUpxMW5lWl84RzVocmJFQ3JfZlpMd3RCZEo5RWZ0MzIxNWV6bHlwdWJJWXhoaWxlM2FHSjBhWG14Sk94ZV96cXFvU1JwWDdKZldmZWdvdWVKdXVfaS1jZjdENXQzSzNyb1d3eWhUMU53QzgxemRiTTlkdFRxZU1OdEN5c1kxOEd2MTJMcnBJWEE0eXdJdFpOYVNMQTNLR292UFlGb0Ztdz0= diff --git a/env-int.env b/env-int.env index a8f67e6f..c6c1d481 100644 --- a/env-int.env +++ b/env-int.env @@ -73,7 +73,7 @@ Connector_AiTavily_API_SECRET = INT_ENC:Z0FBQUFBQnFCdlFlbmRSZVRjTzVKRklFbFgwdVZJ Connector_AiPrivateLlm_API_SECRET = INT_ENC:Z0FBQUFBQnBudkpGSjZ1NWh0aWc1R3Z4MHNaeS1HamtUbndhcUZFZDlqUDhjSmg5eHFfdlVkU0RsVkJ2UVRaMWs3aWhraG5jSlc0YkxNWHVmR2JoSW5ENFFCdkJBM0VienlKSnhzNnBKbTJOUTFKczRfWlQ3bWpmUkRTT1I1OGNUSTlQdExacGRpeXg= Connector_AiMistral_API_SECRET = INT_ENC:Z0FBQUFBQnFCdlFlU2tMLTFnQWhET2Nia2pTcVpBakRaSVFDdUpHRzZ1bkhGVVhMeEVlSnFZU3F3UFRBUkNMMU4tQU92OUdTeDlpM2VZbXJzLURQZ1lPLVB3azgxSDZabkhkSHJ5Y005aWhtcDJzajk3a2JDQUxCZlNKRGw5elJuSzJMUUpTZ2hiSlU= -Service_MSFT_TENANT_ID = common +Service_MSFT_TENANT_ID = organizations # Google Cloud Speech Services configuration Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQnFIc3YtSjhlcklrU2JCOW5mdHFHd0dLTUZZZk9PT3o5RWt5RjAxX2s3ekJRLUUzU0dNSnNseTE4bUpNTnZSTWg0QV9mWm5iX19aWjV4YnRXU1JBSm1INVB5dXNRT2JiYk1tLWRSS29pdTRMdS1lMDZxMkx4VTh3bU5aVWh3cEwyOE1QcXVockgtZWh5bzdNVXQyemFuSmZqRzZZYmNGN21JdjNwNWpPRXB6WU1qSU5rZUVSb3JBS0lhcThvakkwbTRUUHhBdjRZdWNsZ1Z1RmFaNGZLcEpaNVNLdFAxYzFXdTJydU9COWJ0bkNyYUF2X2FNc1BfT05teEs1SE9PeGhPd3VJSFY2VFJ5VEl6V3R3bzd6OTVKTEVRcmt5ZzdBMXBFY1A5dUFJRFJONFBlaDlJcjNBQnBraC0wMTBhNW8wYWZaeHNWclVTOVotLTdWSmVuYzJKcUZSUkdrdXB3VEVESzd4UTI0bGd6SzdCajdoazZXVTVCaGRiaWJaOHg5Z2thSWItcS05U25DbUdrT2M1QV81WEg2dlJfMlBtZU9Bc3V5bmtBWHRoRUVLR2lWNHY3M3hHcU1raFRFOWQwSEtUU1RDWDFRNFlkNHVnTkZDbk5zS3RZeGR2Z015RnRGc3NndFVEQjc4bVpNeE81bXc1MnQ2QjNZeHZCbUJJZVJ2TE5xWEd4M3hHT2hJWW5DOWMxQlNmZE9uMVRGVnRwTUlXZjZCRUZBLU9GWVZGWFpZbUE3WVlpZU1DX1Z0bWQ0bjlaRThHOE9WR3VOVzlYWS1JampTNmxkNmFxWG54WDJjallIT3UyT0tGSzJpeG1tX0JoQjZxbEpESHBhMWZFa205bjdvTVFwSVVidnVzdURZVDAzVVpkekJ2SVZTZmhxQVJ2OWpuRGR2WFE3elMtb3B2ZzhpQVNvRmkzbzRrY1BuamVzM0E2eVM0bXBHTHgtYmhsVG5jNlB1Q1JHZU9HUlNfaTJSQkcwS2FSZnZSOW9oZzdXa1RUVTVTZTgwY01GYXQyQ0xWX1Fnb0xaOTRQY3hTclgweVJ5clc5OVpRWWlDb0JQVXoxVDA0bW8zUE55aGowb1ZZNEpBN2UtSTZTY2llRGhISFFkYWFYVlVBQ0IzbGxzVTQ2V2dsUGV1Y2I5bEZLRnlwdXRHMWZVcnBaTXNzNzNkUVFqR2xnSEQ1VlpTdXpwMFVVYjQ0enFlUnk0d3dDQUtSS1dUVnNyYnBKQW9TRjJxN2JNY2NhRWNONWRpWU5RbzNNZVJBS3EzN2ZMZ1E5VXQtMDFTZklLY1JiSDNYRlFuOF9VYUktS0xoY2IyR0xkT19qTEpIV1p6RFExUWNCQTdqN1kyS0Jaa2lyMDluenc1MS1vdmhPVlE5OUphWEY2dXFYNE04Z3lBUG5DNGZjTUVnYzEzYWhzTHpMdVBzT0dzRGJaT2x5b0pVbWJtUzJxdEd2VGtrc01kTlNPNURoVHhwZzU1d3pTZGJiTUZIME5tQ0xqNWJ2QS1QSEJHV2FEOExHWDByV19rVnc2R2pibnNENEo1cTh4bGNMX2ZpSTBMcjRvQWRhbW5xYVBiZkZzWTRERlVESEU2aHpvdzNMTjlCazRYeEJhMmZwdXY5T25IYkFTaUM3SmdIV1FCX2xxRXctWHZQOHgxLXI1c1JkWmcydkFTUmxFSU03cGtnallnTXplOElQbEJRSEE2aW5KREU0YUxwX25wOFhuS2RIbms1dXNIRHBtNjFtb3B3UGVGb0hwOENKM1hMclBwa3NBa2pFYnZYbEtFbUF0Y3pmeFRmMDNMaTZrR1BZWnBrNUQ1WlU1NVZQSWUxN3dwcXhhcjdXNTl4LVVpYVF3Y0wtRmFyNXZRNTE3UUc2cHVaVVNpaVdHbXRqQVJNZWZmNjdQQ2lwTGd6RFFZN2tSY2NEdmxvaXk4MTZMcmg0VGo3MTN2R2V6cmV3YjdQVlNEZTQySUpaY2pkTHZzUzdJLVJ2WnlOQ3Vmem5FZXRaWjBMWjF4ZEF3ZHJ4VF8tMVNsRnljejVsaEpGOU5JbnhydjNVdzNMOENrWUVsbXp0ZEhuVE1Vd0RJcnp2N0RXUGFuNDM2OXBPbV9LRDUwTWk1NHYwaDhlVEhKUmtEa09INURwNjV5ZE1VWmpRSGdjeXJNc3FqcjZDdmx5WXluNWZ2VlpsWmR2TXVXVnBubEFmQlRfaGRwRndCVXVkMjkyLWVhaDQtZDN1cmFZLUoybGRwbGQ5MTExU2NnZ2lueVNfSjFDQ2NkWGtNX2M1T2I4YnVJOUFueGIxbG1EYlZOcFYtQlE3cm90SE40X0ZjalhLdXM5S2l5aW84ZUJPMlR4MU9EVkhZcHdrX1Zqc0NhWEJacDZHMzQwSzdkdi1Rd2s4Y1dfLS1ES0NfYTNxYl84UTN1S0lIM0pVTTNEYlJ0YW55Tk4yVjBONXNTQWtVZTJ2V3B5eHBJcG9IWGRMMklob0hMbVVZZzJKbTFMUExOQm5HSEZzWHU0VGVIWlJMVzFLeFB0NkkyWFkwWk0wdjdHRmxSWFFoSkJ2Vm5NUWNQQlp6YWlIc2NKLUdhOVVycHd5N3NFMDNVWlAxZGQ1NzRGbm9LcWxEb2tKR1RnVEtvRUc1d3l4aU1IOUQ5RldUT3Z0a3lpRHpVSWJ4MjU4RWY5MEpCQ0VFdHNMbnkxOGswcE44QzJwNXFCVGpIa0VGc2VNXy1qdzVNRU9DaXg2MW9VX3FjUk41QVFVLURwVGFLRTkyNWlENy1IcGZjNW9wY0Y5Q3d5eFg5emVUUF9hV3ZTQWNaNEN0VzdJRlFBR0picXJoUERacWNLbDZhTE8wdWlfZ3kxd2QzOXBOZV9uaUNGMkNJbGhNd3k0S2t3dTRGWVVxTTFRRlg3Ui1zLW1FLU1Mai1yaURjb2Fob2c4MDUyRHN5aldUVWMxLTVNbm5VQTdrYy0zLVFyOHRkNzZ3dGdhbXZXN3JHNkdfZ2RuRXFDM3R2TVB1cDNOdWZGTmpFNnNFTmMxTmFuZDdJUld5bERyQkJ0TGZXRk54NEdqN09hSmVMYV91NXUwNXFvMl9KV0hBNlB4bklNQ2U5WGZLUTdlX2dJenVGcDYwWHBsdTNpbE5mWGhWeXFuUkFPV0puR2h0RkhrR2MwTzJGUmp4bUR6UFlUWTlNbTJLa19hTUZZR0dscVpBbFBReTBRMDNseXo4SXNnZWt4VFdpOERqLV9ZczRkR0QwRFJQM0pqdHluWktDUlp6WU9XSjVNZi1tYnNzcVlGTDRFMzNlSmRTazFfTkNxSjAwM0wxNk9Sd2h1SWpfOW5MVWMtVXYyYlVZR0VuaHRpN1pnNnpHME5raVBMd2h2dDRyMV8yZGFJNnlkcmhtSWdmNlpLN19NcjNkc002dXFxQzhTaDZzRlgzNUJ1SzVpVnp6NVU1Y2luUlM4UEJoajNTOUJadnE1MlhzV0kxSzBObXkteVhNM3RKYW9heDVWWFJ1NGlDM0l0elRPbThwUU9oYkVkbC1PZFNLSHY3WHJiZWpEamNIVC00MlNNWV9qcHdjNDRjRlVhZXlrLTlicVBNaDlDeXdRb0Fwc3RmUGFvbURQZ29yckliaS1VUDNxcXVlYTJJRUhXNUVobk1KUDhHZE16UzBLeDViYVRwZWY3d2w0d253eEZYcExKRGpsaGlBUElaTzB3eUVadnROX1dabENGb3R4ZF9aS05KY0dHTVZaYzRFc1Z4TlZGbFd2NjdYRzJMTzVwU2NaN1Y3MzQ2Z2pzV2RSMzJBbjg0MEhaZmhoREloY0oxOFdjNDZNdVZfYlRKU1Q1M2hYdHgwUjVsTV9USjZCZXlQTTdNRWc3bUxOcXRDVkpTdnJxR0hkWWpaRUdrOEFyNHk4MENwVzdob0hUSkJvam4zZW1kcGxZUjg0RXFRNnBxSUg1MDVHdHRwVlFkWWhHM0ZyZVFvMF96R2V5YjBuMnVZTU5CQ3pVci16SGJlQTQtbnFLa1E2eHFncUg3UmYyYlZvOF82a3d2ZE4tbmxIUlNYYjlrck9QYk5CcV9faXludS1yem1JNjFBdVYyb21RQWFMMFkxX0s1TjQ4czZ2WXI3X0FzRWdNTlZndHl4bnVOTHl2YlZfaURQV053dHl4N1czRFdzaVFnRHB0MWRDV2ZuU2lzX1NZZkRQYzhsT3ItZWw0dVJlVmtFWUM5cEppOGxuYVdpQkN5dV9hQ2dodTJvV3REVkw2dVVDaGtvc0Zqd0V2dldLZEVNRVRRNVRUVmw5aHZmZEpHdk1wS0xwRFc5Vmx4dTdfdGZDRUtCU29qdEVIOW5VdjBmeGpFMFZHSUthamtVN1E2bDZqaEFackVSQnZMN0tyaUhIcUs1ZHMzMzl2TnhadGIwZW5QNS1BM3pSODY3WVFsLU1jeUpCMG1PWmhPVT0= diff --git a/env-prod.env b/env-prod.env index 686f784b..3e55aca8 100644 --- a/env-prod.env +++ b/env-prod.env @@ -72,7 +72,7 @@ Connector_AiTavily_API_SECRET = PROD_ENC:Z0FBQUFBQnFGdnVLcTlLSFJ5b0gwRmJLMFB5MzA Connector_AiPrivateLlm_API_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGanZ6U3pzZWkwXzVPWGtIQ040XzFrTXc5QWRnazdEeEktaUJ0akJmNnEzbWUzNHczLTJfc2dIdzBDY0FTaXZYcDhxNFdNbTNtbEJTb2VRZ0ZYd05hdlNLR1h6SUFzVml2Z1FLY1BjTl90UWozUGxtak1URnhhZmNDRWFTb0dKVUo= Connector_AiMistral_API_SECRET = PROD_ENC:Z0FBQUFBQnFGdnVLRHplbzNheDhIdndsU0xUeGlBYVVXWDRzOF9Tek41WjEtSmNqbnVHRXFaZ0dramlfZWlQelpJWVh5T0F2azBaQWU3ajU0TWljaGpMeTlra0g0LVhKeTRKNGxKY0ZqSkxwdTJLdWM5cWdMVC1TVkpLb2lPdHhyeWtieFJFOHdkVy0= -Service_MSFT_TENANT_ID = common +Service_MSFT_TENANT_ID = organizations # Google Cloud Speech Services configuration Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQnFIc3YtNDZzenJuZEZiQnVMOWRmZjl3R29QOWZRaGlPdk56WG1DR0FSZU5DM3dENWdoMmRpaks1U1VDNDJkZ3d3UXhSbXlkZ2h3SGZfdk54WXVidF82VkdJQXZiRTk0UlhZaUY1b2kwNzNPSm52VFdsdkwtaHJBb2dpRDBVLXRwd19Bb0dUZDkyV1VWZDJ1TG5mZ0ktYXpuS3U1U0JkZUk5TXpMdnhOaUtMN3BIb0pEZ1N0SlpFN3NNby15VTRfWWtxaF9DYjlJcnVKb0ZualVMTUx2aVNGY0JJdE1oZy1xSVBUZDF1aDM0TGVlTzVrNkFHcjlhcEk0SmRIMTFGdDFTMVUxX1dERk9NTXZMb0tVTFRoc20xME1uRkdVV0Z5N200ZTQzSjVsVExoa2VRZmFBU21ZczF0Vm9Ib3BZM2ZneDkwak12UmFyWWd0eng3ZVVFTUFLVzNOazcxeUhLVWUxcEFIZWtNRi1mT29kM1pqNGJJUUh3UVBlNGY3SlotOWZFUk5aQXFXcUFVdnUzc0Z5bERXYUNPbG14VnBNenFvb2tiQ3lZeHNHUVBlQTdTdVdXOEkxaGxCX016WWktWmN2WFcwM0VmVHdvMHVnY212VFE2cjJwUjdENkFCZF9GcUktWWpmWlNXNWVTMHBPdzVxRi15d3FSRDFra2k0NEFmTmpUeVh3SHRuZWE3WGJ4eUNIcE5tdnRqX2NCZnJoMEI2emU4U0ZYN1Nmdlhva1NacFo3UFh3WnpSdGw5ZmNpSGhicFo0ZThReXl3LW9vUzZaMkFHX2lJalFEMWtjZVdqbVpIZGk0cEdEU01TMl9xQkdSNDllTS1GV3lXS0xROTJvSlhaTjlXenJhQ3lOd2p0VjR5ZjEyektUZGJ3UThJOVJuMzhsTTVBVW9BcDFtcjk5Y0pVeW0zX3R0Nk81R3VDRWEzZnRqSXhFUW5ONHFTSWlwQU4yazlDb01KYlFQRjBFVTljdEJIY29WdF9hUkRJOThVTVFfWlJQUXI0Z3RzWFlzR1ZxUWFBd2I1SW1EMWlKdVprT3dKYTlaREp6TkZEZmVsZGEyalZGc3dHaUkyamdmQWtUT2czNzBCZEg0Vk1HSHFpRnhRYzBRNnN3TFkyaE9uMTVXN1VJTmJwbTNUMTdZbVRyc2d6Yl9aaVBXNmFvanROQVhfbWpXTDRlR1RfbklnYnJUQTZPX2JfNnlrWDVDUWJ4Z3YwNXVsTkJFQlRhTG5DVHpwejdsMGl1bzRfRXRTU2dmb3BVMUo4VkQwa0hsTmFBZnVjVzRrQmNzS2R0ZHNGV24yQnktWENtMUp6eG1MQW1ENE1vWFpFUF9PMEpWZVlxX05hSW1QUGlVT1l3MFp4bDBDZVVldHlEUlVCY1VvVlBNTlBhWFlmcVRobDNqRHo0QjZvNDBqVUVKN3JOb2dtYXQxSWw5NERSeEVRdHNUWndzUkY5RjdBOG1FZFRiVTNVSzl5bDNwdTl2SVd5aW5Ub2Q1YlBDRnpBUDkteU44YnV5X05ONmNndm9teUpqaFZVcVlHdGVRcXRpZkJLVnRuMTJSUFhGWndibExqRW03YUJTWXZXUXJ5WXlvd01ISDFuUFpaMFJzNFVQbWRUb2h1Zi1rcXJXMkRQSUFPeWFJN3lzOFc1d3BjWG1kbWlQWGUwelNiSnJXbUpnajdlQTlQR19XNTF0Q3JYcUMzaGp3eU0yZGhKa3FtX0tleHBfekZaWlRJRlZlSzNDVU56cml0TnFJeUc3b09uYVlwbGxFVFR6WFJVMzRmak5yWjBhcjl5ZmJpQ3hpajRXV1dwbDF5N25tNnI2bWtFem1TS08yV3JybUF0enYxRXpkUVdTNVp4WVB0aldJUUN3TnhHcHdMczh5MTFETzNWLXZFSktsdU1vM1JSNXhraDlJRDl0MEhvR1NOQWRaQW1NdzhpZnFVa1hvdXNwY2FvaThHQjVMOXdySnNIcWJlWERfLXVOcHhpN2ZZOW4yVzB3VTI2a3hvVmFkc29aX2ZUZkY5bi04WEV4MTlxNXQ4cTcwaHE4X3hDWkQxelRwSUl2amZOQ0JXRlJjRFhJNVhjNjRmaXp5eG15LTN1MFRvN3BHTFRZQ1ZFVFYyNUxleFpKTHlIVzRnVHk1Y3ZUbV9RUDdqN1Z2M2ZqVG8wa2RoVHJPeENFRDNHV0wwdi1DbEdOVDFJZnRiZGEydlZyM2tQVExOVlo3LXhIUnhZUnB6a2UzZXNtTjR0S2NzUmFNOWNiSHhHTnJDWHowWk1tbVFKUC14M25aQ1hyYjhJM2pxOEtZY0J1WTZrU3l6cDJOdk5iSXpBUk41MFFVellVZFU4UWVDZXFkQnJFbGxQX2J0S3pReU8zZUdsZUgtTnJuSlpfTjdxR3UxWTBEV0JaRV93eE9qa2dNa2tVTHRxMWNyeUh2VWNrYkdKM3BZOURkUlBxUDA3R2M4NnlMTVR2dmNMZi1lZlhzalRJWlFocGRleVRJYXBBY2hCXzFGZEU4ZVFxbHNic3RDV2FYN1dNaWpkaGdwYTEzRkZYRlEtRXR1cERHdnJKX1Zzb1Q0MnVYZkVhb0VYU1JPdFhoV29TMlhTaEppR1lTTURLYmZnNS1pSzl4T1k5MXJ0YV9qX0ZyQ1R6RFFzRndrTW9IUVlxcG5jcTEyYVU3dkpIR0tZZTZiOXNIRFpIalRtUDFBLVNyd1NfNUMtLW52NVpFZGpQenJCOGw0UlJZNlZVT1ZXTm92R3k4c3hTQXFoNFE3TUFHcjRWc01zT082anJZT0laakl5VUk1WDdDaWlubjIwS3RNcjBjTTdpbUNxSmxNR05JaWtEQURlS1h6N2h0NE9CcW5rQ3NXWkwyNXVBUU5mLTU5MG8xX29xZ0t6Z2pKWmhMNG1BNXBhYWkzY0loSmluUXNKdURwQWRIV2laM2dHQTFxV19lbkZXWmdfWEdiWEZsMGVIWDdoMnJ5dzM0ZGtBM3BSRVp2QzFNbFJSWXBManN5WmFVMlp6aUpWMF9jMTRPbWptM1lsTE41NG1kUW4tT0ZqTzNaZnZ5ZzBLZzNNc1N1X2FMMVJ0N3o4a25LMkxKVUE0dTNhU3hZX3RFMUtKcEgtX1B0cTdEMmYyMzdPaEhoeWhaUGRITC11NzRWYTJnZldiUkFvdG95a1RwWnNKaERkT0kxN1RJMzZQZzFiSjl1SlJieTJjaHBMYmZDUlhTT2hvQnRPaTNhS3NzaVc1Tms0X0FyUHRsSXdCLW1OUWk1RkRKc3pqSjVQTFFROEN5M3pxUGVjZHI4SVM3Qmx1S1A2bEEzNWlVWkFndGpUSm4wcV9jRjQ5T0l1c3ZqN0w3Z1dMV2ZtbU9MbTVSOXphX3VLMko2ZEs3U0NIaFFIMVFIcnN0OGIxSjdxNGlHUHRnOEJDaGwzcXJYNFBnOGdFSVFuSGUyOWJ3WmtlVGhGQWk0THdZd1hUbGRydk83SWVzWUJrb21tSlNvVkJjdWYtcWo0aEc1Ri1XNTZoSENaRWJISmp3UlJNMU9vSnNzZ0VudXpxMDA3aGdfSDBNZlA0Y1gybkF4dGl6SzFOc1VMN0dzVkQxVllkSDhyby12SWNxTFRYdThJUm13S3p3cGFYc05TbVc2YVNtZEdCOFBCUXhadkIzNmdkbXpnc1pLYUhzOEtsY2kxVmNYZm9wOS1LOERLRHJhY2VhanNjaThUZW1rS01wUW05SFJxOGd1VF9STlJZWDRiTV92dXlQTkdxN3BYYTN1SUhRSjRNTy1PZWpGd0xhUlVES0hiWE5LUkM5dHNvenR3TVMySC1ueUZXUkxFY2VyRmhISGc2U2ZxeXY2VkJULV9pOTU1QkI5VUNndnVQcVItTW96VTBqRTdzem1IQ1UxVWtWdjhvTERFeGJ6M3dJNERUV1BTeUlRcG1fbUVjQ0lNREF5QkpLeHJHRkFxQS1kZEE4bXJ2aVVSckVoTkZwNGtoRElIcUktQjA1bkNRclM4dWlqUVRXXzdlQ0VjQWZGSTZlR01NQmU5bHQ3bGNtZWU1eHVvRVdQRVU4Rmx0OFRTaWF3cGgyeFJoM25sRk1GNXJtdEpfcEJmYVFrZXd4eXl0c0ZKVjQ3MkFNRjh5bDBTbFZNd256dmxpQlo5Z1FRM1ZmVTJSb3VrZTk3cXVQYmZ6SnNUWGhlSUhrUjVWUHFwemNmbW1scWVxTkcxT1p5dVlvUjhCSVJaSnBjU0dpc3YzVkt1WUtrd2xoQlVNQXh1eDhmTXNISWMyUnBUMmIwamxlS0tjMVRiWDlBcE03b1BHR1FmdmlsX2ZlMTNCaFNvNG1TeTNiQXRNZ2Y1eE1IaFAxTUZGZ1YyZjEzTG9PaGRCdHJzVlB5Mm12T1NiX2RyT2d2RERCRWFHT0dadW5DZjNtdXE4cHhEQlpub2l3bz0= diff --git a/modules/auth/oauthProviderConfig.py b/modules/auth/oauthProviderConfig.py index 713e356e..18554883 100644 --- a/modules/auth/oauthProviderConfig.py +++ b/modules/auth/oauthProviderConfig.py @@ -46,6 +46,21 @@ def msftDataScopesForRefresh() -> str: return " ".join(msftDataScopes) +# Microsoft — Resource ".default": pulls exactly the permissions already +# admin-consented for the app in the user's tenant. Triggers NO interactive / +# admin consent (errors AADSTS65001 only if consent is truly missing), which is +# what we want for tenants that have disabled user consent but granted tenant-wide +# admin consent. msftAuthScopes / msftDataScopes stay as documentation of the +# expected permission set. +MSFT_GRAPH_RESOURCE = "https://graph.microsoft.com" + + +def msftGraphDefaultScopes() -> list: + """Single resource ``.default`` scope for Microsoft Graph (must not be mixed + with individual scopes or reserved scopes — MSAL adds openid/profile/offline_access).""" + return [f"{MSFT_GRAPH_RESOURCE}/.default"] + + # Infomaniak intentionally has no OAuth scope set: the kDrive + Mail data APIs # are only reachable with manually issued Personal Access Tokens (see # wiki/d-guides/infomaniak-token-setup.md). The OAuth /authorize endpoint at diff --git a/modules/auth/tokenRefreshService.py b/modules/auth/tokenRefreshService.py index 7cb3ddeb..70239cee 100644 --- a/modules/auth/tokenRefreshService.py +++ b/modules/auth/tokenRefreshService.py @@ -21,7 +21,9 @@ class TokenRefreshService: def __init__(self): self.rate_limit_map = {} # Track refresh attempts per connection - self.max_attempts_per_hour = 3 + # Allow enough proactive refreshes per hour so the wider pre-expiry window + # (see proactive_refresh) is never blocked for an actively used connection. + self.max_attempts_per_hour = 6 self.refresh_window_minutes = 60 def _is_rate_limited(self, connection_id: str) -> bool: @@ -215,8 +217,13 @@ class TokenRefreshService: async def proactive_refresh(self, user_id: str) -> Dict[str, Any]: """ - Proactively refresh tokens that expire within 5 minutes - + Proactively refresh tokens that expire within the refresh window (30 min). + + A wide window means any request during the last 30 minutes of a token's + lifetime renews it via the refresh token, so an actively used connection + effectively never lapses (the stored expiresAt always reflects the real + Microsoft/Google token lifetime — it is never faked). + Args: user_id: User ID to check tokens for @@ -241,7 +248,7 @@ class TokenRefreshService: failed_count = 0 rate_limited_count = 0 current_time = getUtcTimestamp() - five_minutes = 5 * 60 # 5 minutes in seconds + refresh_window = 30 * 60 # 30 minutes in seconds (matches TokenManager.getFreshToken) # Process each connection for connection in connections: @@ -250,9 +257,9 @@ class TokenRefreshService: connection.tokenExpiresAt and connection.authority in [AuthAuthority.GOOGLE, AuthAuthority.MSFT]): - # Check if token expires within 5 minutes + # Check if token expires within the refresh window time_until_expiry = connection.tokenExpiresAt - current_time - if 0 < time_until_expiry <= five_minutes: + if 0 < time_until_expiry <= refresh_window: # Check rate limiting if self._is_rate_limited(connection.id): diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 6325cfa8..33d2d1c3 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -34,7 +34,7 @@ from modules.auth import ( clearRefreshTokenCookie, ) from modules.auth.tokenManager import TokenManager -from modules.auth.oauthProviderConfig import msftAuthScopes, msftDataScopes, msftDataScopesForRefresh +from modules.auth.oauthProviderConfig import msftDataScopes, msftDataScopesForRefresh, msftGraphDefaultScopes from modules.auth.homeMandateService import ensureHomeMandate from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp from modules.shared.i18nRegistry import apiRouteContext @@ -123,7 +123,7 @@ def auth_login(request: Request) -> RedirectResponse: ) state_jwt = _issue_oauth_state({"flow": _FLOW_LOGIN}) auth_url = msal_app.get_authorization_request_url( - scopes=msftAuthScopes, + scopes=msftGraphDefaultScopes(), redirect_uri=AUTH_REDIRECT_URI, state=state_jwt, prompt="select_account", @@ -155,7 +155,7 @@ async def auth_login_callback( ) token_response = msal_app.acquire_token_by_authorization_code( code, - scopes=msftAuthScopes, + scopes=msftGraphDefaultScopes(), redirect_uri=AUTH_REDIRECT_URI, ) if "error" in token_response: @@ -320,16 +320,22 @@ def auth_connect( request: Request, connectionId: str = Query(..., description="UserConnection id"), connectTicket: str = Query(..., description="Short-lived ticket from POST /api/connections/{id}/connect"), - reauth: Optional[int] = Query(0, description="If 1, force the consent screen so newly added scopes are granted"), + reauth: Optional[int] = Query(0, description="If 1, force account re-selection (prompt=select_account). Never forces consent."), ) -> RedirectResponse: """Start Microsoft Data OAuth for an existing connection. Authenticated via ``connectTicket`` (issued by POST connect) so the popup works when the UI uses Bearer tokens in localStorage instead of cookies. - With ``reauth=1`` the consent screen is forced (``prompt=consent``) so the - user re-grants permissions and any newly added scopes (e.g. Calendars.Read, - Contacts.Read) actually land on the access token. + We never force ``prompt=consent``: with the Graph ``.default`` scope the + tenant's admin-consented permissions (incl. newly added scopes) are pulled + automatically. Forcing consent would re-trigger an interactive consent that + admin-restricted scopes (Sites.ReadWrite.All, Mail.Send, …) escalate to + "Need admin approval" for non-admin users. Genuinely missing permissions + instead surface as AADSTS65001, which routes to the admin-consent flow. + + With ``reauth=1`` we only force account re-selection so the user can pick a + different account when refreshing a connection. """ try: _require_msft_data_config() @@ -351,10 +357,16 @@ def auth_connect( login_kwargs["domain_hint"] = login_hint.split("@", 1)[1] login_kwargs["prompt"] = "login" if reauth: - login_kwargs["prompt"] = "consent" + # Refreshing a connection: let the user re-pick the account, but never + # force consent — prompt=consent would escalate admin-restricted scopes + # to "Need admin approval" for non-admin users even though tenant-wide + # admin consent already covers them via the ".default" scope. + login_kwargs["prompt"] = "select_account" + # ".default" pulls exactly the tenant-consented Graph permissions and + # avoids re-triggering the admin-consent screen for external tenants. auth_url = msal_app.get_authorization_request_url( - scopes=msftDataScopes, + scopes=msftGraphDefaultScopes(), redirect_uri=DATA_REDIRECT_URI, **login_kwargs, ) @@ -389,7 +401,7 @@ async def auth_connect_callback( ) token_response = msal_app.acquire_token_by_authorization_code( code, - scopes=msftDataScopes, + scopes=msftGraphDefaultScopes(), redirect_uri=DATA_REDIRECT_URI, ) if "error" in token_response: @@ -608,24 +620,17 @@ def adminconsent_callback( status_code=400, ) - if not state: - logger.error("Admin consent success callback missing state") - return HTMLResponse( - content=""" - - Admin Consent Failed - -

Admin Consent Failed

-

Parameter „state“ fehlt.

- - - """, - status_code=400, - ) - - state_data = _parse_oauth_state(state) - if state_data.get("flow") != "admin_consent": - raise HTTPException(status_code=400, detail=routeApiMsg("Invalid OAuth flow for this callback")) + # When admin consent is initiated outside our /adminconsent route (e.g. + # the "Grant admin consent" button in the Azure portal, or a raw consent + # URL), Microsoft redirects without our state JWT. The consent itself is + # still recorded server-side, so we must not hard-fail — only validate the + # flow claim when a state is actually present. + if state: + state_data = _parse_oauth_state(state) + if state_data.get("flow") != "admin_consent": + raise HTTPException(status_code=400, detail=routeApiMsg("Invalid OAuth flow for this callback")) + else: + logger.warning("Admin consent callback without state — accepting (consent initiated outside our route)") granted = str(admin_consent or "").strip().lower() in ("true", "1", "yes") if not granted: